diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index 3af6001c9d..6f422e751f 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -4677,6 +4677,7 @@ dependencies = [ "codex-otel", "codex-plugin", "codex-protocol", + "codex-realtime-webrtc", "codex-rollout", "codex-sandboxing", "codex-shell-command", diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index a810a48c17..63f59d5408 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -76,6 +76,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; use tokio::sync::Mutex; +use tokio::sync::Semaphore; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tracing::debug; @@ -129,6 +130,37 @@ enum RealtimeFanoutTaskStop { Detach, } +/// Serializes turn admission with the safety cutoff for one voice session. +struct RealtimeHandoffAdmission { + gate: Semaphore, + retired: AtomicBool, +} + +impl RealtimeHandoffAdmission { + fn new() -> Self { + Self { + gate: Semaphore::new(1), + retired: AtomicBool::new(false), + } + } + + async fn route(&self, session: &Arc, text: String) { + let Ok(_permit) = self.gate.acquire().await else { + return; + }; + if !self.retired.load(Ordering::Acquire) { + session.route_realtime_text_input(text).await; + } + } + + async fn retire(&self) { + let Ok(_permit) = self.gate.acquire().await else { + return; + }; + self.retired.store(true, Ordering::Release); + } +} + pub(crate) struct RealtimeConversationManager { state: Mutex>, mode_instructions: Mutex>, @@ -467,6 +499,8 @@ struct ConversationState { input_task: JoinHandle<()>, fanout_task: Option>, realtime_active: Arc, + // A misalignment failure retires only this session's queued handoffs. + route_handoffs: Arc, stop_token: CancellationToken, } @@ -489,6 +523,7 @@ struct RealtimeStart { struct RealtimeStartOutput { realtime_active: Arc, + route_handoffs: Arc, events_rx: Receiver, transcript_tail_rx: Receiver, sdp: Option, @@ -529,6 +564,20 @@ impl RealtimeConversationManager { ) } + /// Prevent already parsed handoffs from starting turns after a safety failure. + /// Keep the gate on the old session so a later voice session starts fresh. + pub(crate) async fn retire_handoffs_for_misalignment(&self) { + let route_handoffs = self + .state + .lock() + .await + .as_ref() + .map(|state| Arc::clone(&state.route_handoffs)); + if let Some(route_handoffs) = route_handoffs { + route_handoffs.retire().await; + } + } + async fn start( &self, start: RealtimeStart, @@ -581,6 +630,7 @@ impl RealtimeConversationManager { let (transcript_tail_tx, transcript_tail_rx) = async_channel::bounded::(1); let realtime_active = Arc::new(AtomicBool::new(true)); + let route_handoffs = Arc::new(RealtimeHandoffAdmission::new()); let stop_token = CancellationToken::new(); let handoff = RealtimeHandoffState { output_tx: handoff_output_tx, @@ -690,10 +740,12 @@ impl RealtimeConversationManager { input_task: task, fanout_task: None, realtime_active: Arc::clone(&realtime_active), + route_handoffs: Arc::clone(&route_handoffs), stop_token, }); Ok(RealtimeStartOutput { realtime_active, + route_handoffs, events_rx, transcript_tail_rx, sdp, @@ -1571,6 +1623,7 @@ async fn handle_start_inner( let RealtimeStartOutput { realtime_active, + route_handoffs, events_rx, transcript_tail_rx, sdp, @@ -1597,10 +1650,9 @@ async fn handle_start_inner( match &event { RealtimeEvent::AudioOut(_) => {} _ => { - info!( - event = ?event, - "received realtime conversation event" - ); + // Transcripts, handoffs, and conversation items may contain + // credentials. Keep this receipt free of event payloads. + info!("received realtime conversation event"); } } if let RealtimeEvent::Error(_) = &event { @@ -1613,9 +1665,9 @@ async fn handle_start_inner( _ => None, }; if let Some(text) = maybe_routed_text { - debug!(text = %text, "[realtime-text] realtime conversation text output"); - let sess_for_routed_text = Arc::clone(&sess_clone); - sess_for_routed_text.route_realtime_text_input(text).await; + // The routed text can contain spoken prompts or workspace secrets. + debug!("[realtime-text] realtime conversation text output"); + route_handoffs.route(&sess_clone, text).await; } sess_clone .send_event_raw(ev(EventMsg::RealtimeConversationRealtime( @@ -1626,7 +1678,7 @@ async fn handle_start_inner( .await; } if let Ok(text) = transcript_tail_rx.recv().await { - sess_clone.route_realtime_text_input(text).await; + route_handoffs.route(&sess_clone, text).await; } if fanout_realtime_active.swap(false, Ordering::Relaxed) { match end { @@ -1785,7 +1837,7 @@ pub(crate) async fn handle_speech( sub_id: String, params: ConversationSpeechParams, ) { - debug!(text = %params.text, "[realtime-text] appending realtime speech"); + debug!("[realtime-text] appending realtime speech"); if let Err(err) = sess.conversation.append_speech(params.text).await { error!("failed to append realtime speech: {err}"); if sess.conversation.running_state().await.is_some() { diff --git a/codex-rs/core/src/session/turn.rs b/codex-rs/core/src/session/turn.rs index 4719e1d891..7f31667988 100644 --- a/codex-rs/core/src/session/turn.rs +++ b/codex-rs/core/src/session/turn.rs @@ -636,6 +636,12 @@ pub(crate) async fn run_turn( } Err(e) => { info!("Turn error: {e:#}"); + if matches!( + e.details(), + CodexErrorDetails::MisalignmentPolicyViolation { .. } + ) { + sess.conversation.retire_handoffs_for_misalignment().await; + } let error = e.to_codex_protocol_error(); sess.emit_turn_error_lifecycle(turn_context.as_ref(), error.clone()) .await; diff --git a/codex-rs/core/tests/suite/mod.rs b/codex-rs/core/tests/suite/mod.rs index 1c6a629926..1c23e2aba9 100644 --- a/codex-rs/core/tests/suite/mod.rs +++ b/codex-rs/core/tests/suite/mod.rs @@ -128,6 +128,7 @@ mod prompt_debug_tests; mod quota_exceeded; mod realtime_conversation; mod realtime_initial_items; +mod realtime_misalignment; mod realtime_sideband_endpoint; mod reasoning_effort_override; mod remote_env; diff --git a/codex-rs/core/tests/suite/realtime_misalignment.rs b/codex-rs/core/tests/suite/realtime_misalignment.rs new file mode 100644 index 0000000000..109e61b78c --- /dev/null +++ b/codex-rs/core/tests/suite/realtime_misalignment.rs @@ -0,0 +1,151 @@ +//! A failed turn must not admit handoffs still arriving from its voice session. + +use anyhow::Context; +use anyhow::Result; +use codex_config::config_toml::RealtimeWsVersion; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ConversationStartParams; +use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::Op; +use codex_protocol::protocol::RealtimeConversationRealtimeEvent; +use codex_protocol::protocol::RealtimeEvent; +use codex_protocol::protocol::RealtimeOutputModality; +use core_test_support::responses; +use core_test_support::skip_if_no_network; +use core_test_support::test_codex::test_codex; +use core_test_support::wait_for_event_match; +use futures::SinkExt; +use pretty_assertions::assert_eq; +use serde_json::json; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::sync::oneshot; +use tokio_tungstenite::accept_async; +use tokio_tungstenite::tungstenite::Message; +use wiremock::ResponseTemplate; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn misalignment_retires_late_voice_handoff_before_it_starts_a_turn() -> Result<()> { + skip_if_no_network!(Ok(())); + + let api_server = responses::start_mock_server().await; + let first_response = responses::mount_response_once( + &api_server, + ResponseTemplate::new(403).set_body_json(json!({ + "error": { + "message": "This request violated the misalignment policy.", + "type": "invalid_request_error", + "code": "misalignment_policy_violation" + } + })), + ) + .await; + + let listener = TcpListener::bind("127.0.0.1:0").await?; + let realtime_url = format!("ws://{}", listener.local_addr()?); + let (late_handoff_tx, mut late_handoff_rx) = oneshot::channel(); + let (finish_tx, finish_rx) = oneshot::channel(); + let sideband = tokio::spawn(async move { + let (stream, _) = listener.accept().await?; + let mut websocket = accept_async(stream).await?; + websocket + .send(Message::Text( + json!({ + "type": "session.updated", + "session": {"id": "session_misalignment", "instructions": "backend prompt"} + }) + .to_string() + .into(), + )) + .await?; + for (handoff_id, text) in [ + ("initial_handoff", "first request"), + ("late_handoff", "must not start a turn"), + ] { + if handoff_id == "late_handoff" { + (&mut late_handoff_rx).await?; + } + websocket + .send(Message::Text( + json!({ + "type": "conversation.handoff.requested", + "handoff_id": handoff_id, + "item_id": handoff_id, + "input_transcript": text + }) + .to_string() + .into(), + )) + .await?; + } + finish_rx.await?; + websocket.close(None).await?; + Ok::<_, anyhow::Error>(()) + }); + + let mut builder = test_codex().with_config(move |config| { + config.experimental_realtime_ws_base_url = Some(realtime_url); + config.realtime.version = RealtimeWsVersion::V1; + }); + let test = builder.build_with_auto_env(&api_server).await?; + test.codex + .submit(Op::RealtimeConversationStart(ConversationStartParams { + client_managed_handoffs: false, + delegation_ack_filler: None, + flush_transcript_tail_on_session_end: false, + codex_responses_as_items: false, + codex_response_item_prefix: None, + codex_response_handoff_mode: + codex_protocol::protocol::CodexResponseHandoffMode::Thinking, + codex_response_handoff_channel_prefixes: None, + model: None, + output_modality: RealtimeOutputModality::Audio, + include_startup_context: true, + initial_items: Vec::new(), + realtime_start_instructions: None, + realtime_end_instructions: None, + prompt: Some(Some("backend prompt".to_string())), + realtime_session_id: None, + transport: None, + version: None, + voice: None, + })) + .await?; + + wait_for_event_match(&test.codex, |event| match event { + EventMsg::Error(error) + if error.codex_error_info == Some(CodexErrorInfo::MisalignmentPolicyViolation) => + { + Some(()) + } + _ => None, + }) + .await; + assert_eq!(first_response.requests().len(), 1); + late_handoff_tx.send(()).expect("sideband still open"); + wait_for_event_match(&test.codex, |event| match event { + EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::HandoffRequested(handoff), + }) if handoff.handoff_id == "late_handoff" => Some(()), + _ => None, + }) + .await; + finish_tx.send(()).expect("sideband still open"); + sideband.await??; + + // The fanout forwards a handoff only after deciding whether to route it. + // Give any spawned turn a chance to reach the mock API before checking. + tokio::time::sleep(Duration::from_millis(200)).await; + let requests = api_server + .received_requests() + .await + .context("API requests")?; + assert_eq!( + requests + .iter() + .filter(|request| request.url.path() == "/v1/responses") + .count(), + 1, + ); + Ok(()) +} diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rs index 371c51bdfc..5f5138ecff 100644 --- a/codex-rs/realtime-webrtc/src/client.rs +++ b/codex-rs/realtime-webrtc/src/client.rs @@ -19,6 +19,22 @@ use crate::message_reader::MessageReader; const DEADLINE: Duration = Duration::from_secs(/*secs*/ 5); const RUNTIME_INITIALIZATION_DEADLINE: Duration = Duration::from_secs(/*secs*/ 30); +/// Startup failures eligible for recovery remain distinct from all other failures. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConnectionError { + NegotiationTimedOut, + Failed, +} +impl std::fmt::Display for ConnectionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::NegotiationTimedOut => "voice negotiation timed out", + Self::Failed => "voice connection failed", + }) + } +} +impl std::error::Error for ConnectionError {} + /// Owns one helper. Dropping it terminates the process and leaves its waiter to reap it. /// A successful handshake establishes compatibility only, not an active audio session. pub struct VoiceHost { @@ -94,6 +110,12 @@ impl VoiceHost { Duration::from_secs(/*secs*/ 20), ) .await?; + if response == (Message::TransportTimedOut {}) { + self.process.terminate(); + // A lost exit notification or failed cleanup is not retryable. + timeout(DEADLINE, &mut self.exit).await??; + return Err(ConnectionError::NegotiationTimedOut.into()); + } ensure!( response == Message::TransportReady {}, "unexpected voice helper response" diff --git a/codex-rs/realtime-webrtc/src/client_tests.rs b/codex-rs/realtime-webrtc/src/client_tests.rs index b5256bee47..e0c96ab7f3 100644 --- a/codex-rs/realtime-webrtc/src/client_tests.rs +++ b/codex-rs/realtime-webrtc/src/client_tests.rs @@ -61,3 +61,52 @@ async fn cancelling_initialization_terminates_the_owned_helper() -> anyhow::Resu tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 1), spawned.exit_rx).await??; Ok(()) } + +#[cfg(unix)] +#[tokio::test] +async fn negotiation_timeout_waits_for_helper_exit_before_allowing_retry() -> anyhow::Result<()> { + let spawned = codex_utils_pty::spawn_pipe_process( + std::path::Path::new("/bin/sleep"), + &["30".to_owned()], + std::path::Path::new("/"), + &child_environment(std::iter::empty()), + /*arg0*/ &None, + &[], + ) + .await?; + let (output, receiver) = tokio::sync::mpsc::channel(/*buffer*/ 1); + output + .send(crate::encode_frame(&crate::Message::TransportTimedOut {})?) + .await?; + let (reaped, exit) = tokio::sync::oneshot::channel(); + let host = super::VoiceHost { + process: spawned.session, + output: crate::message_reader::MessageReader::new(receiver), + exit, + }; + let answer = crate::SessionDescription::try_from("synthetic-answer".to_owned()).unwrap(); + let mut connection = Box::pin(host.apply_answer(answer)); + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(connection.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + let code = + tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 1), spawned.exit_rx).await??; + // The process has exited, but the explicit reaping acknowledgement is still required. + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(connection.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + reaped.send(code).unwrap(); + let error = connection + .await + .err() + .expect("timeout must not return a connected helper"); + assert_eq!( + error.downcast_ref::(), + Some(&crate::ConnectionError::NegotiationTimedOut) + ); + Ok(()) +} diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rs index c65d724755..8abfa53eb3 100644 --- a/codex-rs/realtime-webrtc/src/lib.rs +++ b/codex-rs/realtime-webrtc/src/lib.rs @@ -3,6 +3,7 @@ mod message_reader; mod protocol; mod session; +pub use client::ConnectionError; pub use client::VoiceHost; pub use protocol::AudioControls; pub use protocol::AudioState; diff --git a/codex-rs/realtime-webrtc/src/protocol.rs b/codex-rs/realtime-webrtc/src/protocol.rs index f94bf53bae..2859aaea87 100644 --- a/codex-rs/realtime-webrtc/src/protocol.rs +++ b/codex-rs/realtime-webrtc/src/protocol.rs @@ -81,6 +81,7 @@ pub enum Message { Offer { sdp: SessionDescription }, ApplyAnswer { sdp: SessionDescription }, TransportReady {}, + TransportTimedOut {}, OpenDevices {}, DevicesOpened {}, SetAudioControls { controls: AudioControls }, diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rs index 1241a40fa5..b699c03e4c 100644 --- a/codex-rs/realtime-webrtc/src/session.rs +++ b/codex-rs/realtime-webrtc/src/session.rs @@ -26,7 +26,10 @@ const ANSWER_WAIT: Duration = Duration::from_secs(/*secs*/ 40); static RUNTIME: Mutex>> = Mutex::new(None); enum Command { - Answer(SessionDescription, blocking::SyncSender<()>), + Answer( + SessionDescription, + blocking::SyncSender>, + ), Controls(AudioControls), } @@ -196,15 +199,22 @@ impl RealtimeWebrtcSessionHandle { } /// Called off the UI thread; return only after negotiation, device startup and restored controls. - pub fn apply_answer_sdp(&self, answer: String) -> Result<()> { - let sdp = SessionDescription::try_from(answer).map_err(anyhow::Error::msg)?; + pub fn apply_answer_sdp( + &self, + answer: String, + ) -> std::result::Result<(), crate::ConnectionError> { + let sdp = + SessionDescription::try_from(answer).map_err(|_| crate::ConnectionError::Failed)?; let (complete, result) = blocking::sync_channel(/*bound*/ 1); - self.send(Command::Answer(sdp, complete))?; - if result.recv_timeout(ANSWER_WAIT).is_err() { - self.close(); - anyhow::bail!("voice connection failed"); + self.send(Command::Answer(sdp, complete)) + .map_err(|_| crate::ConnectionError::Failed)?; + match result.recv_timeout(ANSWER_WAIT) { + Ok(result) => result, + Err(_) => { + self.close(); + Err(crate::ConnectionError::Failed) + } } - Ok(()) } pub fn take_error(&self) -> Option { @@ -254,13 +264,23 @@ async fn run( biased; command = commands.recv() => match command { Some(Command::Answer(sdp, complete)) if !connected => { - host = host.apply_answer(sdp).await?.open_devices().await?; + host = match host.apply_answer(sdp).await { + Ok(host) => host, + Err(error) => { + let failure = error.downcast_ref::() + .copied().unwrap_or(crate::ConnectionError::Failed); + let _ = complete.send(Err(failure)); + // This failure is delivered by the startup completion only. + return Ok(()); + } + }; + host = host.open_devices().await?; let applied = startup_controls(&mut commands, controls, |initial| { host.begin_audio_controls(initial) })?; applied.await?; connected = true; - let _ = complete.send(()); + let _ = complete.send(Ok(())); } Some(Command::Controls(next)) => { if connected { diff --git a/codex-rs/realtime-webrtc/src/session_tests.rs b/codex-rs/realtime-webrtc/src/session_tests.rs index b3f32e432c..a5afc9b912 100644 --- a/codex-rs/realtime-webrtc/src/session_tests.rs +++ b/codex-rs/realtime-webrtc/src/session_tests.rs @@ -180,3 +180,21 @@ fn startup_dispatch_is_ordered_but_acknowledgement_does_not_block_setters() { ); drop(applied); } + +#[tokio::test] +async fn startup_completion_preserves_classified_failure() { + for failure in [ + crate::ConnectionError::NegotiationTimedOut, + crate::ConnectionError::Failed, + ] { + let (handle, mut commands, _) = handles(); + let caller = handle.clone(); + let thread = std::thread::spawn(move || caller.apply_answer_sdp("synthetic-answer".into())); + let Some(Command::Answer(_, complete)) = commands.recv().await else { + panic!("answer expected") + }; + complete.send(Err(failure)).unwrap(); + assert_eq!(thread.join().unwrap(), Err(failure)); + assert_eq!(handle.take_error(), None); + } +} diff --git a/codex-rs/tui/Cargo.toml b/codex-rs/tui/Cargo.toml index 040f44f502..f72429a992 100644 --- a/codex-rs/tui/Cargo.toml +++ b/codex-rs/tui/Cargo.toml @@ -53,6 +53,8 @@ codex-models-manager = { workspace = true } codex-otel = { workspace = true } codex-plugin = { workspace = true } codex-protocol = { workspace = true } +codex-realtime-webrtc = { workspace = true } +futures = { workspace = true } codex-rollout = { workspace = true } codex-sandboxing = { workspace = true } codex-shell-command = { workspace = true } diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e7140a6533..64e4f4038c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -235,6 +235,7 @@ mod pets; mod platform_actions; mod plugin_mentions; mod rate_limit_refresh; +mod realtime_delivery; mod recap; mod reconnect; mod replay_filter; @@ -617,6 +618,10 @@ pub(crate) struct App { windows_sandbox: WindowsSandboxState, thread_event_channels: HashMap, + pending_realtime_speech_replay: HashMap>, + pending_realtime_transcript_replay: + HashMap>, + realtime_replay_order: VecDeque, temporary_structured_requests: HashMap>, /// Track title generation across thread switches and deduplicate automatic requests. pending_thread_titles: HashSet<(ThreadId, ThreadTitleDestination)>, diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs index cc16390c17..b77998c8f9 100644 --- a/codex-rs/tui/src/app/event_dispatch.rs +++ b/codex-rs/tui/src/app/event_dispatch.rs @@ -21,7 +21,7 @@ use codex_app_server_protocol::ThreadGoalStatus; #[cfg(target_os = "windows")] use codex_config::types::WindowsSandboxModeToml; -const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2); +pub(super) const SHUTDOWN_FIRST_EXIT_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 2); impl App { pub(super) async fn handle_event( @@ -785,6 +785,23 @@ impl App { .apply_reserve_fallback_to_pending_turn(&mut op); } let is_user_turn = matches!(&op, AppCommand::UserTurn { .. }); + let is_realtime_stop = matches!(&op, AppCommand::RealtimeConversationStop { .. }); + let realtime_stop_thread_id = match &op { + AppCommand::RealtimeConversationStop { thread_id } => Some(*thread_id), + _ => None, + }; + let realtime_speech_delivery_id = match &op { + AppCommand::RealtimeConversationSpeech { delivery_id, .. } => { + Some(*delivery_id) + } + _ => None, + }; + let is_realtime_conversation = matches!( + &op, + AppCommand::RealtimeConversationStart { .. } + | AppCommand::RealtimeConversationStop { .. } + | AppCommand::RealtimeConversationSpeech { .. } + ); if is_user_turn { let screen_size = tui.terminal.last_known_screen_size; self.handle_draw_pre_render(tui, screen_size)?; @@ -797,6 +814,10 @@ impl App { } self.chat_widget.prepare_local_op_submission(&op); if let Err(err) = self.submit_active_thread_op(app_server, op).await { + if let Some(delivery_id) = realtime_speech_delivery_id { + self.chat_widget + .restore_undelivered_realtime_speech(delivery_id); + } if self.recover_transport_error(&err) { return Ok(AppRunControl::Continue); @@ -817,10 +838,22 @@ impl App { && self .chat_widget .handle_turn_start_rejection(format!("Failed to start turn: {err:#}")); - if !handled { + if is_realtime_conversation { + let message = format!("Voice conversation failed: {err:#}"); + if is_realtime_stop { + if self.chat_widget.thread_id() == realtime_stop_thread_id { + self.chat_widget.reset_realtime_conversation(); + self.chat_widget.add_error_message(message); + } + } else { + self.chat_widget.on_realtime_error(message); + } + tracing::error!(error = ?err, "realtime conversation request failed"); + } else if handled { + tracing::error!(error = ?err, "failed to start turn through app server"); + } else { return Err(err); } - tracing::error!(error = ?err, "failed to start turn through app server"); } } AppEvent::ConfirmSafetyBufferedRetry { @@ -1656,6 +1689,47 @@ impl App { self.sync_active_thread_personality_setting(app_server, personality) .await; } + AppEvent::RealtimeWebrtcOfferCreated { + thread_id, + attempt_id, + result, + } => { + if self.chat_widget.thread_id() == Some(thread_id) { + self.chat_widget + .on_realtime_webrtc_offer_created(thread_id, attempt_id, result); + } else if let Ok(offer) = result { + offer.handle.close(); + } + } + AppEvent::RealtimeWebrtcConnected { + thread_id, + attempt_id, + result, + } => { + if self.chat_widget.thread_id() == Some(thread_id) { + self.chat_widget + .on_realtime_webrtc_connected(attempt_id, result); + } + } + AppEvent::StopRealtimeConversation { thread_id } => { + match tokio::time::timeout( + SHUTDOWN_FIRST_EXIT_TIMEOUT, + app_server.thread_realtime_stop(thread_id), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!( + %thread_id, + %error, + "failed to stop voice conversation after switching threads" + ), + Err(_) => tracing::warn!( + %thread_id, + "timed out stopping voice conversation after switching threads" + ), + } + } AppEvent::SettingsSelectionClosed => { self.app_event_tx.send(AppEvent::SettingsSelectionSettled); } @@ -3369,6 +3443,7 @@ impl App { }) } ExitMode::Immediate => { + self.stop_realtime_conversation(app_server).await; self.pending_shutdown_exit_thread_id = None; AppRunControl::Exit(ExitReason::UserRequested) } diff --git a/codex-rs/tui/src/app/misalignment_policy.rs b/codex-rs/tui/src/app/misalignment_policy.rs index 68e4dbed08..3f3dbad870 100644 --- a/codex-rs/tui/src/app/misalignment_policy.rs +++ b/codex-rs/tui/src/app/misalignment_policy.rs @@ -4,6 +4,7 @@ use super::*; use crate::app_server_session::turn_permissions_overrides; use crate::chatwidget::MisalignmentReview; +use crate::chatwidget::MisalignmentTurnSource; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::TurnStartResponse; @@ -161,6 +162,10 @@ impl App { Ok(response) => { let store = &self.ensure_thread_channel(review.thread_id).store; store.lock().await.active_turn_id = Some(response.turn.id.clone()); + self.chat_widget.clear_misalignment_for_new_turn( + &response.turn.id, + MisalignmentTurnSource::AcknowledgedContinuation, + ); self.chat_widget.handle_server_notification( ServerNotification::TurnStarted( codex_app_server_protocol::TurnStartedNotification { diff --git a/codex-rs/tui/src/app/realtime_delivery.rs b/codex-rs/tui/src/app/realtime_delivery.rs new file mode 100644 index 0000000000..c0c1ba9dae --- /dev/null +++ b/codex-rs/tui/src/app/realtime_delivery.rs @@ -0,0 +1,392 @@ +//! Preserve voice answers and accepted transcripts across thread-widget replacement. +//! +//! A queued speech command can outlive its widget. Retain its bounded final +//! item by original thread until replay either renders that item or needs a +//! fallback, without adding history to whichever thread is currently visible. + +use super::*; +use codex_protocol::models::MessagePhase; +use std::collections::HashMap; +use std::collections::HashSet; + +type ReplayedVoiceTextCounts = HashMap<(String, String), usize>; +const MAX_INACTIVE_REALTIME_REPLAY_THREADS: usize = 16; + +fn normalized_replay_text(text: &str) -> String { + text.split_whitespace().collect::>().join(" ") +} + +fn record_voice_item( + item: &ThreadItem, + turn_id: &str, + delegated: &mut HashSet, + items: &mut HashMap<(String, String, &'static str), String>, +) { + match item { + ThreadItem::UserMessage { id, content, .. } => { + if let Some(input) = crate::chatwidget::realtime_delegation_input(content) { + delegated.insert(turn_id.to_string()); + let display = crate::chatwidget::realtime_delegation_display_text(input); + items.insert( + (turn_id.to_string(), id.clone(), "user"), + normalized_replay_text(&display), + ); + } + } + ThreadItem::AgentMessage { + id, + text, + phase: Some(MessagePhase::FinalAnswer) | None, + .. + } if delegated.contains(turn_id) + && !crate::chatwidget::is_private_realtime_agent_item(item) + && !text.trim().is_empty() => + { + items.insert( + (turn_id.to_string(), id.clone(), "assistant"), + normalized_replay_text(text), + ); + } + _ => {} + } +} + +pub(super) fn replayed_voice_texts_from_turns(turns: &[Turn]) -> ReplayedVoiceTextCounts { + let mut delegated = HashSet::new(); + let mut items = HashMap::new(); + for turn in turns { + for item in &turn.items { + record_voice_item(item, &turn.id, &mut delegated, &mut items); + } + } + count_replayed_voice_texts(items) +} + +fn count_replayed_voice_texts( + items: HashMap<(String, String, &'static str), String>, +) -> ReplayedVoiceTextCounts { + let mut counts = HashMap::new(); + for ((_, _, role), text) in items { + if !text.is_empty() { + *counts.entry((role.to_string(), text)).or_insert(0) += 1; + } + } + counts +} + +pub(super) fn replayed_voice_texts(snapshot: &ThreadEventSnapshot) -> ReplayedVoiceTextCounts { + let mut delegated = snapshot + .delegated_turns + .iter() + .cloned() + .collect::>(); + let mut items = HashMap::new(); + for turn in &snapshot.turns { + for item in &turn.items { + record_voice_item(item, &turn.id, &mut delegated, &mut items); + } + } + for event in &snapshot.events { + let ThreadBufferedEvent::Notification(notification) = event else { + continue; + }; + match notification.as_ref() { + ServerNotification::ItemStarted(n) => { + if let ThreadItem::UserMessage { content, .. } = &n.item + && crate::chatwidget::realtime_delegation_input(content).is_some() + { + delegated.insert(n.turn_id.clone()); + } + } + ServerNotification::ItemCompleted(n) => { + record_voice_item(&n.item, &n.turn_id, &mut delegated, &mut items); + } + ServerNotification::TurnCompleted(n) if n.turn.status == TurnStatus::Completed => { + if let Some(item) = n.turn.items.iter().rev().find(|item| { + matches!( + item, + ThreadItem::AgentMessage { + phase: Some(MessagePhase::FinalAnswer) | None, + .. + } + ) && !crate::chatwidget::is_private_realtime_agent_item(item) + }) { + record_voice_item(item, &n.turn.id, &mut delegated, &mut items); + } + } + _ => {} + } + } + count_replayed_voice_texts(items) +} + +impl App { + pub(super) fn note_realtime_replay_thread(&mut self, thread_id: ThreadId) { + if self.realtime_replay_order.contains(&thread_id) { + return; + } + if self.realtime_replay_order.len() >= MAX_INACTIVE_REALTIME_REPLAY_THREADS + && let Some(oldest) = self.realtime_replay_order.pop_front() + { + self.pending_realtime_transcript_replay.remove(&oldest); + self.pending_realtime_speech_replay.remove(&oldest); + } + self.realtime_replay_order.push_back(thread_id); + } + + pub(super) fn forget_realtime_replay_thread(&mut self, thread_id: ThreadId) { + self.realtime_replay_order + .retain(|saved| *saved != thread_id); + self.pending_realtime_transcript_replay.remove(&thread_id); + self.pending_realtime_speech_replay.remove(&thread_id); + } + + pub(super) fn retain_inactive_realtime_transcript( + &mut self, + thread_id: ThreadId, + notification: &ServerNotification, + ) { + let (role, text, complete) = match notification { + ServerNotification::ThreadRealtimeTranscriptDelta(n) => { + (n.role.as_str(), n.delta.as_str(), false) + } + ServerNotification::ThreadRealtimeTranscriptDone(n) => { + (n.role.as_str(), n.text.as_str(), true) + } + _ => return, + }; + if !matches!(role, "user" | "assistant") { + return; + } + self.note_realtime_replay_thread(thread_id); + let records = self + .pending_realtime_transcript_replay + .entry(thread_id) + .or_default(); + if complete { + if text.trim().is_empty() { + if let Some(index) = records + .iter() + .rposition(|record| record.role == role && !record.complete) + { + records.remove(index); + } + return; + } + let mut bounded = text.to_string(); + if bounded.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES { + let mut end = crate::chatwidget::MAX_TRANSCRIPT_BYTES; + while !bounded.is_char_boundary(end) { + end -= 1; + } + bounded.truncate(end); + } + if let Some(partial) = records + .iter_mut() + .rev() + .find(|record| record.role == role && !record.complete) + { + partial.text = bounded; + partial.complete = true; + return; + } + if records.len() >= crate::chatwidget::MAX_REPLAY_TRANSCRIPT_CELLS { + records.pop_front(); + } + records.push_back(crate::chatwidget::RealtimeTranscriptRecord { + role: role.to_string(), + text: bounded, + complete: true, + }); + } else { + if text.is_empty() { + return; + } + let partial = records + .iter_mut() + .rev() + .find(|record| record.role == role && !record.complete); + if let Some(partial) = partial { + partial.text.push_str(text); + if partial.text.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES { + let mut start = partial.text.len() - crate::chatwidget::MAX_TRANSCRIPT_BYTES; + while !partial.text.is_char_boundary(start) { + start += 1; + } + partial.text.drain(..start); + } + } else { + let mut bounded = text.to_string(); + if bounded.len() > crate::chatwidget::MAX_TRANSCRIPT_BYTES { + let mut start = bounded.len() - crate::chatwidget::MAX_TRANSCRIPT_BYTES; + while !bounded.is_char_boundary(start) { + start += 1; + } + bounded.drain(..start); + } + if records.len() >= crate::chatwidget::MAX_REPLAY_TRANSCRIPT_CELLS { + records.pop_front(); + } + records.push_back(crate::chatwidget::RealtimeTranscriptRecord { + role: role.to_string(), + text: bounded, + complete: false, + }); + } + } + } + + pub(super) fn retain_realtime_replay_state_before_replace(&mut self) { + if let Some(thread_id) = self.chat_widget.thread_id() { + if self.chat_widget.may_receive_realtime_transcripts() { + self.note_realtime_replay_thread(thread_id); + // Even a session without captions can receive a final queued caption + // after this widget has been replaced. + self.pending_realtime_transcript_replay + .entry(thread_id) + .or_default(); + } + let cells = self.chat_widget.take_realtime_transcript_cells_for_replay(); + if !cells.is_empty() { + self.note_realtime_replay_thread(thread_id); + let pending = self + .pending_realtime_transcript_replay + .entry(thread_id) + .or_default(); + // Only the active widget produces these cells; reattachment consumes them + // before another widget for this thread can become active. + // Up to 32 accepted cells plus one partial for each speaker. + debug_assert!(pending.len() + cells.len() <= 34); + pending.extend(cells); + } + } + for (thread_id, turn_id, item) in self + .chat_widget + .take_undelivered_realtime_speech_for_replay() + { + self.note_realtime_replay_thread(thread_id); + let pending = self + .pending_realtime_speech_replay + .entry(thread_id) + .or_default(); + // The active widget is the only producer for a thread; selecting + // that thread reconciles these records before it can produce more. + debug_assert!(pending.len() < 16); + pending.push((turn_id, item)); + } + } + + pub(super) fn restore_realtime_replay_state_after_replay( + &mut self, + replayed_final_items: &HashMap<(String, String), String>, + mut replayed_voice_texts: ReplayedVoiceTextCounts, + ) { + let Some(thread_id) = self.chat_widget.thread_id() else { + return; + }; + self.realtime_replay_order + .retain(|saved| *saved != thread_id); + let mut retained_assistant_captions = HashMap::::new(); + if let Some(mut cells) = self.pending_realtime_transcript_replay.remove(&thread_id) { + cells.retain(|record| { + if !record.complete { + return true; + } + let key = (record.role.clone(), normalized_replay_text(&record.text)); + if let Some(count) = replayed_voice_texts.get_mut(&key) + && *count > 0 + { + *count -= 1; + return false; + } + true + }); + for record in &cells { + if record.complete && record.role == "assistant" { + *retained_assistant_captions + .entry(normalized_replay_text(&record.text)) + .or_default() += 1; + } + } + self.chat_widget.restore_realtime_transcript_cells(cells); + } + let Some(pending) = self.pending_realtime_speech_replay.remove(&thread_id) else { + return; + }; + for (turn_id, item) in pending { + let ThreadItem::AgentMessage { id, text, .. } = &item else { + continue; + }; + // A matching ID with empty text did not render the saved answer. + // Any nonempty snapshot text is authoritative even if it changed. + if replayed_final_items + .get(&(turn_id.clone(), id.clone())) + .is_none_or(|text| text.trim().is_empty()) + { + // A completed caption restored above already displays this + // answer. Consume only one exact match; partial or distinct + // captions must not hide an undelivered fallback. + let spoken_text = text.trim().strip_prefix("[FINAL]").unwrap_or(text); + if let Some(count) = + retained_assistant_captions.get_mut(&normalized_replay_text(spoken_text)) + && *count > 0 + { + *count -= 1; + continue; + } + self.chat_widget + .render_undelivered_realtime_speech(thread_id, turn_id, item); + } + } + } +} + +pub(super) fn completed_agent_items( + snapshot: &ThreadEventSnapshot, +) -> HashMap<(String, String), String> { + let mut completed = completed_agent_items_from_turns(&snapshot.turns); + for event in &snapshot.events { + let ThreadBufferedEvent::Notification(notification) = event else { + continue; + }; + match notification.as_ref() { + ServerNotification::ItemCompleted(completed_item) => { + if let ThreadItem::AgentMessage { id, text, .. } = &completed_item.item + && !text.trim().is_empty() + { + completed.insert((completed_item.turn_id.clone(), id.clone()), text.clone()); + } + } + ServerNotification::TurnCompleted(completed_turn) + if completed_turn.turn.status == TurnStatus::Completed => + { + for item in &completed_turn.turn.items { + if let ThreadItem::AgentMessage { id, text, .. } = item + && !text.trim().is_empty() + { + completed + .insert((completed_turn.turn.id.clone(), id.clone()), text.clone()); + } + } + } + _ => {} + } + } + completed +} + +pub(super) fn completed_agent_items_from_turns( + turns: &[Turn], +) -> HashMap<(String, String), String> { + let mut completed = HashMap::new(); + for turn in turns { + for item in &turn.items { + if let ThreadItem::AgentMessage { id, text, .. } = item + && !text.trim().is_empty() + { + completed.insert((turn.id.clone(), id.clone()), text.clone()); + } + } + } + completed +} diff --git a/codex-rs/tui/src/app/session_lifecycle.rs b/codex-rs/tui/src/app/session_lifecycle.rs index 8f2a0d804f..2240d773d5 100644 --- a/codex-rs/tui/src/app/session_lifecycle.rs +++ b/codex-rs/tui/src/app/session_lifecycle.rs @@ -484,6 +484,7 @@ impl App { /// This helper copies every known nickname/role from `AgentNavigationState` into the /// replacement widget so that replayed collab items render agent names immediately. pub(super) fn replace_chat_widget(&mut self, mut chat_widget: ChatWidget) { + self.retain_realtime_replay_state_before_replace(); chat_widget.cyber_policy_notice = self.chat_widget.cyber_policy_notice.clone(); self.commit_animation = None; // Transfer the last-written terminal title to the replacement widget @@ -581,6 +582,16 @@ impl App { return Ok(()); } let previous_thread_id = self.active_thread_id; + // Notifications already routed to the active channel must reach the old + // widget before its transcript state is transferred. Events routed after + // deactivation are retained separately for that thread. + if let Some(mut receiver) = self.active_thread_rx.take() { + while let Ok(event) = receiver.try_recv() { + self.handle_thread_event_now_recovering_file_changes(event) + .await; + } + self.active_thread_rx = Some(receiver); + } self.store_active_thread_receiver().await; self.active_thread_id = None; let Some((receiver, mut snapshot)) = self.activate_thread_for_replay(thread_id).await @@ -656,6 +667,10 @@ impl App { self.recap.seed_from_progress(recap_progress, now); self.schedule_recap_check(thread_id, now); + // The old widget owns the local helper and its undelivered answers. + // Transfer replay state before stopping its backend voice session. + self.retain_realtime_replay_state_before_replace(); + self.stop_realtime_conversation(app_server).await; self.render_thread_snapshot(tui, app_server, thread_id, snapshot, !is_replay_only)?; if is_replay_only && self @@ -749,6 +764,9 @@ impl App { pub(super) fn reset_thread_event_state(&mut self) { self.abort_all_thread_event_listeners(); self.thread_event_channels.clear(); + self.pending_realtime_speech_replay.clear(); + self.pending_realtime_transcript_replay.clear(); + self.realtime_replay_order.clear(); self.pending_server_profiles.clear(); self.agents_overview.activity.clear(); self.agent_navigation.clear(); diff --git a/codex-rs/tui/src/app/side.rs b/codex-rs/tui/src/app/side.rs index 9dca8bb7f7..db616b3aa2 100644 --- a/codex-rs/tui/src/app/side.rs +++ b/codex-rs/tui/src/app/side.rs @@ -524,6 +524,7 @@ impl App { } else { self.refresh_pending_thread_approvals().await; } + self.forget_realtime_replay_thread(thread_id); self.sync_active_agent_label(); } diff --git a/codex-rs/tui/src/app/startup.rs b/codex-rs/tui/src/app/startup.rs index 2bb22a51df..13626e677f 100644 --- a/codex-rs/tui/src/app/startup.rs +++ b/codex-rs/tui/src/app/startup.rs @@ -707,6 +707,9 @@ See the Codex keymap documentation for supported actions and examples." pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), + pending_realtime_speech_replay: HashMap::new(), + pending_realtime_transcript_replay: HashMap::new(), + realtime_replay_order: VecDeque::new(), temporary_structured_requests: HashMap::new(), pending_thread_titles: HashSet::new(), thread_event_listener_tasks: HashMap::new(), diff --git a/codex-rs/tui/src/app/test_support.rs b/codex-rs/tui/src/app/test_support.rs index f750ff70df..26932ed15c 100644 --- a/codex-rs/tui/src/app/test_support.rs +++ b/codex-rs/tui/src/app/test_support.rs @@ -64,6 +64,9 @@ pub(super) async fn make_test_app() -> App { pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), + pending_realtime_speech_replay: HashMap::new(), + pending_realtime_transcript_replay: HashMap::new(), + realtime_replay_order: VecDeque::new(), temporary_structured_requests: HashMap::new(), pending_thread_titles: HashSet::new(), thread_event_listener_tasks: HashMap::new(), diff --git a/codex-rs/tui/src/app/tests.rs b/codex-rs/tui/src/app/tests.rs index bd98ce8a6c..988e979b2a 100644 --- a/codex-rs/tui/src/app/tests.rs +++ b/codex-rs/tui/src/app/tests.rs @@ -42,6 +42,10 @@ mod patch_approval_tests; mod permission_shortcuts_tests; mod plugin_catalog; mod rate_limits; +#[path = "tests/realtime_handoff_e2e.rs"] +mod realtime_handoff_e2e; +#[path = "tests/realtime_start.rs"] +mod realtime_start; #[path = "tests/recap_generation_tests.rs"] mod recap_generation; mod safety_buffering; @@ -1334,6 +1338,7 @@ async fn replayed_turn_complete_submits_restored_queued_follow_up() { while new_op_rx.try_recv().is_ok() {} app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![ThreadBufferedEvent::Notification(Box::new( @@ -1387,6 +1392,7 @@ async fn replay_only_thread_keeps_restored_queue_visible() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![ThreadBufferedEvent::Notification(Box::new( @@ -1438,6 +1444,7 @@ async fn replay_thread_snapshot_keeps_queue_when_running_state_only_comes_from_s app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![], @@ -1487,6 +1494,7 @@ async fn replay_thread_snapshot_in_progress_turn_restores_running_queue_state() app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: vec![test_turn("turn-1", TurnStatus::InProgress, Vec::new())], events: Vec::new(), @@ -1516,6 +1524,7 @@ async fn replay_thread_snapshot_in_progress_turn_restores_running_state_without_ app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: vec![test_turn("turn-1", TurnStatus::InProgress, Vec::new())], events: Vec::new(), @@ -1558,6 +1567,7 @@ async fn replay_thread_snapshot_does_not_submit_queue_before_replay_catches_up() app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![ @@ -1699,6 +1709,7 @@ async fn replay_thread_snapshot_restores_collaboration_mode_for_draft_submit() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![], @@ -1779,6 +1790,7 @@ async fn replay_thread_snapshot_restores_collaboration_mode_without_input() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![], @@ -1829,6 +1841,7 @@ async fn replayed_interrupted_turn_restores_queued_input_to_composer() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![ThreadBufferedEvent::Notification(Box::new( @@ -3714,6 +3727,7 @@ async fn replay_snapshot_with_pending_request_suppresses_replay_notices() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session(thread_id, test_path_buf("/tmp/main"))), turns: Vec::new(), events: vec![ @@ -4096,6 +4110,7 @@ async fn replayed_file_change_approval_recovers_snapshot_changes() { let cwd = test_path_buf("/tmp/project").abs(); app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session(thread_id, cwd.clone().into_path_buf())), turns: vec![test_turn( "turn-replayed-approval", @@ -5053,6 +5068,7 @@ async fn side_thread_snapshot_does_not_refresh_from_fork_history() { .insert(side_thread_id, SideThreadState::new(parent_thread_id)); let snapshot = ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(ThreadSessionState { rollout_path: None, ..test_thread_session(side_thread_id, test_path_buf("/tmp/side")) @@ -5081,6 +5097,7 @@ async fn side_thread_snapshot_skips_session_header_preamble() { .insert(side_thread_id, SideThreadState::new(parent_thread_id)); let snapshot = ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(ThreadSessionState { forked_from_id: Some(parent_thread_id), fork_parent_title: None, @@ -5256,6 +5273,7 @@ async fn active_side_thread_renders_live_mcp_startup_notifications() { app.activate_thread_channel(side_thread_id).await; app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session( side_thread_id, test_path_buf("/tmp/side"), @@ -5839,6 +5857,9 @@ async fn make_test_app() -> App { pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), + pending_realtime_speech_replay: HashMap::new(), + pending_realtime_transcript_replay: HashMap::new(), + realtime_replay_order: VecDeque::new(), temporary_structured_requests: HashMap::new(), pending_thread_titles: HashSet::new(), thread_event_listener_tasks: HashMap::new(), @@ -5933,6 +5954,9 @@ pub(super) async fn make_test_app_with_channels() -> ( pending_shutdown_exit_thread_id: None, windows_sandbox: WindowsSandboxState::default(), thread_event_channels: HashMap::new(), + pending_realtime_speech_replay: HashMap::new(), + pending_realtime_transcript_replay: HashMap::new(), + realtime_replay_order: VecDeque::new(), temporary_structured_requests: HashMap::new(), pending_thread_titles: HashSet::new(), thread_event_listener_tasks: HashMap::new(), @@ -8049,6 +8073,7 @@ async fn replay_thread_snapshot_replays_turn_history_in_order() { let thread_id = ThreadId::new(); app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session( thread_id, test_path_buf("/home/user/project"), @@ -8165,6 +8190,7 @@ async fn replace_chat_widget_reseeds_collab_agent_metadata_for_replay() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: None, turns: Vec::new(), events: vec![ThreadBufferedEvent::Notification(Box::new( @@ -8240,6 +8266,7 @@ async fn refreshed_snapshot_session_persists_resumed_turns() { ..initial_session.clone() }; let mut snapshot = ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(initial_session), turns: Vec::new(), events: Vec::new(), diff --git a/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs b/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs index 72f1a7fda3..765278421f 100644 --- a/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs +++ b/codex-rs/tui/src/app/tests/advanced_reasoning_tests.rs @@ -65,6 +65,7 @@ async fn switching_from_ultra_thread_restores_configured_plan_effort() { app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(ultra_session), turns: Vec::new(), events: Vec::new(), @@ -74,6 +75,7 @@ async fn switching_from_ultra_thread_restores_configured_plan_effort() { ); app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(normal_session), turns: Vec::new(), events: Vec::new(), diff --git a/codex-rs/tui/src/app/tests/disconnect_tests.rs b/codex-rs/tui/src/app/tests/disconnect_tests.rs index 59bb0fd583..c6117fdfa2 100644 --- a/codex-rs/tui/src/app/tests/disconnect_tests.rs +++ b/codex-rs/tui/src/app/tests/disconnect_tests.rs @@ -258,6 +258,38 @@ async fn disconnected_command_center_keeps_input_and_blocks_actions() -> Result< Ok(()) } +#[tokio::test] +async fn remote_disconnect_retires_voice_before_reconnecting() -> Result<()> { + let (mut app, _events, _ops) = make_test_app_with_channels().await; + let thread_id = ThreadId::new(); + app.active_thread_id = Some(thread_id); + app.chat_widget + .handle_thread_session_quiet(test_thread_session(thread_id, app.config.cwd.to_path_buf())); + app.app_server_target = AppServerTarget::Remote { + endpoint: crate::resolve_remote_addr("ws://127.0.0.1:9")?, + }; + crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id); + assert!(app.chat_widget.is_current_realtime_attempt( + thread_id, /*attempt_id*/ 0, /*input_generation*/ 0 + )); + + assert!(app.begin_reconnect()); + assert_eq!( + ( + app.reconnect.offline, + app.chat_widget.is_current_realtime_attempt( + thread_id, /*attempt_id*/ 0, /*input_generation*/ 0 + ), + ), + (true, false) + ); + assert!(app.begin_reconnect()); + assert!(!app.chat_widget.is_current_realtime_attempt( + thread_id, /*attempt_id*/ 0, /*input_generation*/ 0 + )); + Ok(()) +} + // Common framing/bootstrap for both transports. Returning None deliberately loses a reply. pub(super) async fn serve_reconnect_requests( mut socket: tokio_tungstenite::WebSocketStream, diff --git a/codex-rs/tui/src/app/tests/mcp_startup.rs b/codex-rs/tui/src/app/tests/mcp_startup.rs index 40d0922099..ffc6db5cc4 100644 --- a/codex-rs/tui/src/app/tests/mcp_startup.rs +++ b/codex-rs/tui/src/app/tests/mcp_startup.rs @@ -36,6 +36,7 @@ async fn subagent_mcp_startup_settles_while_cached_servers_remain_deferred() { app.activate_thread_channel(subagent_thread_id).await; app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session( subagent_thread_id, test_path_buf("/tmp/subagent"), diff --git a/codex-rs/tui/src/app/tests/patch_approval_tests.rs b/codex-rs/tui/src/app/tests/patch_approval_tests.rs index 3bc8ec323a..c01f6789f3 100644 --- a/codex-rs/tui/src/app/tests/patch_approval_tests.rs +++ b/codex-rs/tui/src/app/tests/patch_approval_tests.rs @@ -287,6 +287,7 @@ async fn replayed_patch_approval_pager_recovers_stored_turn_changes() { let cwd = app.chat_widget.config_ref().cwd.to_path_buf(); app.replay_thread_snapshot( ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(test_thread_session(thread_id, cwd)), turns: vec![test_turn( TURN_ID, diff --git a/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs b/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs new file mode 100644 index 0000000000..305650684b --- /dev/null +++ b/codex-rs/tui/src/app/tests/realtime_handoff_e2e.rs @@ -0,0 +1,270 @@ +//! Real Core-to-app-server-to-TUI handoff coverage without an audio device. + +use super::*; +use crate::chatwidget::UserMessage; +use app_test_support::MockResponsesConfig; +use app_test_support::create_mock_responses_server_sequence_unchecked; +use codex_app_server_client::AppServerEvent; +use codex_app_server_protocol::ServerNotification; +use codex_login::AuthCredentialsStoreMode; +use codex_login::AuthKeyringBackendKind; +use codex_model_provider_info::ModelProviderInfo; +use core_test_support::responses; +use core_test_support::responses::WebSocketConnectionConfig; +use pretty_assertions::assert_eq; +use ratatui::buffer::Buffer; +use ratatui::layout::Rect; +use serde_json::json; +use std::time::Duration; +use tokio::time::timeout; +use wiremock::Mock; +use wiremock::ResponseTemplate; +use wiremock::matchers::method; +use wiremock::matchers::path; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn delegated_core_events_keep_private_output_hidden_and_deliver_final_speech() -> Result<()> { + core_test_support::skip_if_no_network!(Ok(())); + + let mut commentary = + responses::ev_assistant_message("private-commentary", "PRIVATE COMMENTARY"); + commentary["item"]["phase"] = json!("commentary"); + let mut final_answer = responses::ev_assistant_message( + "public-final", + "[ANALYSIS] is the marker you asked about.", + ); + final_answer["item"]["phase"] = json!("final_answer"); + let model_server = create_mock_responses_server_sequence_unchecked(vec![ + responses::sse(vec![ + responses::ev_response_created("delegated-response"), + responses::ev_reasoning_item("private-reasoning", &["PRIVATE REASONING"], &[]), + commentary, + final_answer, + responses::ev_completed("delegated-response"), + ]), + responses::sse(vec![ + responses::ev_response_created("typed-response"), + responses::ev_assistant_message("typed-final", "Typed answer stays visible."), + responses::ev_completed("typed-response"), + ]), + ]) + .await; + Mock::given(method("POST")) + .and(path("/v1/live")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("Location", "/v1/live/rtc_tui_handoff") + .set_body_string("v=answer\r\n"), + ) + .mount(&model_server) + .await; + let realtime_server = + responses::start_websocket_server_with_headers(vec![WebSocketConnectionConfig { + requests: vec![ + vec![ + json!({ + "type": "session.started", + "session": { "id": "sess_tui_handoff", "instructions": "backend prompt" } + }), + json!({ + "type": "delegation.created", + "offset_ms": 100, + "item": { + "id": "delegation_tui", + "type": "delegation", + "target": "client", + "content": [{"type": "input_text", "text": "explain the marker"}] + } + }), + ], + vec![], + vec![], + vec![], + ], + response_headers: Vec::new(), + accept_delay: None, + close_after_requests: false, + }]) + .await; + + let (mut app, mut app_events, mut ops) = make_test_app_with_channels().await; + let codex_home = tempfile::tempdir()?; + MockResponsesConfig::new(&model_server.uri()) + .with_root_config(&format!( + "experimental_realtime_ws_base_url = {:?}\nexperimental_realtime_webrtc_call_base_url = {:?}", + realtime_server.uri(), + format!("{}/v1", model_server.uri()), + )) + .with_extra_config("[realtime]\nversion = \"v3\"\ntype = \"conversational\"") + .write(codex_home.path())?; + codex_login::login_with_api_key( + codex_home.path(), + "sk-test-key", + AuthCredentialsStoreMode::File, + AuthKeyringBackendKind::default(), + )?; + app.config.codex_home = codex_home.path().to_path_buf().abs(); + app.config.sqlite = codex_state::SqliteConfig::new_for_testing(codex_home.path().abs()); + app.config.cli_auth_credentials_store_mode = AuthCredentialsStoreMode::File; + app.config.model = Some("mock-model".to_string()); + app.config.model_provider_id = "mock_provider".to_string(); + app.config.model_provider = ModelProviderInfo { + name: "Mock provider for test".to_string(), + base_url: Some(format!("{}/v1", model_server.uri())), + request_max_retries: Some(0), + stream_max_retries: Some(0), + ..ModelProviderInfo::default() + }; + app.config.experimental_realtime_ws_base_url = Some(realtime_server.uri().to_string()); + app.config.experimental_realtime_webrtc_call_base_url = + Some(format!("{}/v1", model_server.uri())); + + let mut app_server = Box::pin(crate::start_embedded_app_server_for_picker(&app.config)).await?; + let started = app_server.start_thread(&app.config).await?; + let thread_id = started.session.thread_id; + app.enqueue_primary_thread_session(started.session, started.turns) + .await?; + crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id); + while ops.try_recv().is_ok() {} + let mut tui = crate::tui::test_support::make_test_tui()?; + app_server + .thread_realtime_start(thread_id, "v=offer\r\n".to_string(), /*model*/ None) + .await?; + // The sideband fixture sends its delegation after the first outbound request. + app_server + .thread_realtime_append_speech(thread_id, "fixture prompt".to_string()) + .await?; + + let mut rendered = Vec::new(); + let speech = timeout(Duration::from_secs(15), async { + loop { + tokio::select! { + event = app_server.next_event() => { + app.handle_app_server_event( + &app_server, + event.expect("embedded app-server should stay connected"), + ).await; + app.drain_active_thread_events(&mut tui).await?; + } + event = app_events.recv() => { + let event = event.expect("TUI event stream should stay open"); + if let AppEvent::InsertHistoryCell(cell) = &event { + rendered.extend(cell.display_lines(/*width*/ 80).into_iter().map(|line| line.to_string())); + } + app.handle_event(&mut tui, &mut app_server, event).await?; + } + op = ops.recv() => { + let op = op.expect("TUI command stream should stay open"); + if matches!(op, Op::RealtimeConversationSpeech { .. }) { + break Ok::<_, color_eyre::Report>(op); + } + app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(op)).await?; + } + } + } + }) + .await??; + assert!( + matches!(&speech, Op::RealtimeConversationSpeech { text, .. } if text.as_str() == "[ANALYSIS] is the marker you asked about."), + "unexpected speech: {speech:?}" + ); + app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(speech)) + .await?; + assert_eq!( + timeout( + Duration::from_secs(5), + realtime_server.wait_for_request(/*connection_index*/ 0, /*request_index*/ 1), + ) + .await? + .body_json(), + json!({ + "type": "session.context.append", + "content": [{ + "type": "input_text", + "text": "[ANALYSIS] is the marker you asked about." + }], + "channel": "speakable" + }) + ); + let rendered = rendered.join("\n"); + assert!(!rendered.contains("PRIVATE REASONING"), "{rendered}"); + assert!(!rendered.contains("PRIVATE COMMENTARY"), "{rendered}"); + let area = Rect::new( + /*x*/ 0, /*y*/ 0, /*width*/ 80, /*height*/ 24, + ); + let mut buffer = Buffer::empty(area); + app.chat_widget.render(area, &mut buffer); + let visible = buffer + .content() + .iter() + .map(ratatui::buffer::Cell::symbol) + .collect::(); + assert!(!visible.contains("PRIVATE REASONING"), "{visible}"); + assert!(!visible.contains("PRIVATE COMMENTARY"), "{visible}"); + assert!( + ops.try_recv().is_err(), + "a final answer must be sent only once" + ); + + // A typed request on the same live thread must follow the normal visible path. + app.chat_widget + .restore_user_message_to_composer(UserMessage::from("typed follow-up")); + app.chat_widget + .handle_key_event(crossterm::event::KeyEvent::new( + crossterm::event::KeyCode::Enter, + crossterm::event::KeyModifiers::NONE, + )); + let typed_op = next_user_turn_op(&mut ops); + app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(typed_op)) + .await?; + let mut typed_rendered = Vec::new(); + timeout(Duration::from_secs(10), async { + loop { + tokio::select! { + event = app_server.next_event() => { + let event = event.expect("embedded app-server should stay connected"); + let completed = matches!(&event, AppServerEvent::ServerNotification(notification) + if matches!(notification.as_ref(), ServerNotification::TurnCompleted(_))); + app.handle_app_server_event(&app_server, event).await; + app.drain_active_thread_events(&mut tui).await?; + if completed { + break Ok::<_, color_eyre::Report>(()); + } + } + event = app_events.recv() => { + let event = event.expect("TUI event stream should stay open"); + if let AppEvent::InsertHistoryCell(cell) = &event { + typed_rendered.extend(cell.display_lines(/*width*/ 80).into_iter().map(|line| line.to_string())); + } + app.handle_event(&mut tui, &mut app_server, event).await?; + } + op = ops.recv() => { + let op = op.expect("TUI command stream should stay open"); + assert!(!matches!(op, Op::RealtimeConversationSpeech { .. }), "typed answer must not be spoken"); + app.handle_event(&mut tui, &mut app_server, AppEvent::CodexOp(op)).await?; + } + } + } + }) + .await??; + while let Ok(event) = app_events.try_recv() { + if let AppEvent::InsertHistoryCell(cell) = &event { + typed_rendered.extend( + cell.display_lines(/*width*/ 80) + .into_iter() + .map(|line| line.to_string()), + ); + } + app.handle_event(&mut tui, &mut app_server, event).await?; + } + assert!( + typed_rendered + .join("\n") + .contains("Typed answer stays visible."), + "typed final must appear in history: {typed_rendered:?}" + ); + assert!(ops.try_recv().is_err(), "typed answer must not be spoken"); + realtime_server.shutdown().await; + app_server.shutdown().await?; + Ok(()) +} diff --git a/codex-rs/tui/src/app/tests/realtime_start.rs b/codex-rs/tui/src/app/tests/realtime_start.rs new file mode 100644 index 0000000000..76c83a20e8 --- /dev/null +++ b/codex-rs/tui/src/app/tests/realtime_start.rs @@ -0,0 +1,64 @@ +//! Voice startup rejection coverage without an audio device. + +use super::*; +use crate::app::tests::session_lifecycle_requests::recorded_params; +use crate::app::tests::session_lifecycle_requests::start_recording_remote_app_server; + +#[tokio::test] +async fn rejected_voice_start_does_not_leave_local_capture_active() -> Result<()> { + for replay_only in [false, true] { + let (mut app, mut events, _ops) = make_test_app_with_channels().await; + let (mut app_server, requests, proxy) = + start_recording_remote_app_server(&app.config).await?; + let thread_id = ThreadId::new(); + app.chat_widget + .handle_thread_session_quiet(test_thread_session( + thread_id, + app.config.cwd.to_path_buf(), + )); + crate::chatwidget::activate_voice_for_thread(&mut app.chat_widget, thread_id); + assert!(app.chat_widget.may_receive_realtime_transcripts()); + if replay_only { + app.app_server_target = AppServerTarget::Remote { + endpoint: crate::resolve_remote_addr("ws://127.0.0.1:9")?, + }; + app.active_thread_id = Some(thread_id); + app.ensure_thread_channel(thread_id).mark_replay_only(); + assert!(app.thread_unavailable(thread_id)); + } + let mut tui = crate::tui::test_support::make_test_tui()?; + Box::pin(app.handle_event( + &mut tui, + &mut app_server, + AppEvent::CodexOp(Op::RealtimeConversationStart { + thread_id, + offer_sdp: String::from("v=0\r\n").into(), + }), + )) + .await?; + + assert!(!app.chat_widget.may_receive_realtime_transcripts()); + assert!(recorded_params(&requests, "thread/realtime/start").is_empty()); + let rendered = std::iter::from_fn(|| events.try_recv().ok()) + .filter_map(|event| match event { + AppEvent::InsertHistoryCell(cell) => Some( + cell.display_lines(/*width*/ 80) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"), + ), + _ => None, + }) + .collect::>() + .join("\n"); + if replay_only { + insta::assert_snapshot!("rejected_voice_start_replay_only", rendered); + } else { + insta::assert_snapshot!("rejected_voice_start_no_active_thread", rendered); + } + app_server.shutdown().await?; + proxy.await??; + } + Ok(()) +} diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap new file mode 100644 index 0000000000..dcab8f663d --- /dev/null +++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_no_active_thread.snap @@ -0,0 +1,6 @@ +--- +source: tui/src/app/tests/realtime_start.rs +assertion_line: 59 +expression: rendered +--- +■ No active thread is available. diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap new file mode 100644 index 0000000000..fb035f0336 --- /dev/null +++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_start__rejected_voice_start_replay_only.snap @@ -0,0 +1,6 @@ +--- +source: tui/src/app/tests/realtime_start.rs +assertion_line: 57 +expression: rendered +--- +■ This conversation is read-only or unavailable; no operation was sent. diff --git a/codex-rs/tui/src/app/tests/startup.rs b/codex-rs/tui/src/app/tests/startup.rs index 02b3be4e6b..7b822b6df4 100644 --- a/codex-rs/tui/src/app/tests/startup.rs +++ b/codex-rs/tui/src/app/tests/startup.rs @@ -1406,6 +1406,7 @@ async fn ignore_same_thread_resume_allows_retrying_read_only_view() -> Result<() &app_server, thread_id, ThreadEventSnapshot { + delegated_turns: Vec::new(), session: Some(session), turns: vec![test_turn("running", TurnStatus::InProgress, Vec::new())], events: Vec::new(), diff --git a/codex-rs/tui/src/app/thread_events.rs b/codex-rs/tui/src/app/thread_events.rs index e192656dad..f577a1f02a 100644 --- a/codex-rs/tui/src/app/thread_events.rs +++ b/codex-rs/tui/src/app/thread_events.rs @@ -11,6 +11,7 @@ use std::borrow::Cow; #[derive(Debug, Clone)] pub(super) struct ThreadEventSnapshot { pub(super) session: Option, + pub(super) delegated_turns: Vec, pub(super) turns: Vec, pub(super) events: Vec, pub(super) input_state: Option, @@ -24,6 +25,24 @@ pub(super) enum ThreadBufferedEvent { FeedbackSubmission(FeedbackThreadEvent), } +fn is_voice_handoff_item(item: &ThreadItem) -> bool { + matches!(item, ThreadItem::UserMessage { content, .. } + if crate::chatwidget::realtime_delegation_input(content).is_some()) +} + +fn hide_private_items_after_voice_handoff(items: &mut Vec, delegated: bool) { + let has_marker = items.iter().any(is_voice_handoff_item); + // An evicted marker leaves no reliable order, so hide private items + // conservatively. A present marker lets earlier typed output survive. + let mut voice_started = delegated && !has_marker; + items.retain(|item| { + if is_voice_handoff_item(item) { + voice_started = true; + } + !voice_started || !crate::chatwidget::is_private_realtime_agent_item(item) + }); +} + #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct FeedbackThreadEvent { pub(super) category: FeedbackCategory, @@ -53,6 +72,7 @@ pub(super) struct ThreadEventStore { pub(super) capacity: usize, pub(super) active: bool, pub(super) buffered_agent_message_delta_bytes: usize, + delegated_turns: VecDeque, recap_progress: recap::RecapProgress, } @@ -95,6 +115,7 @@ impl ThreadEventStore { capacity, active: false, buffered_agent_message_delta_bytes: 0, + delegated_turns: VecDeque::new(), recap_progress: recap::RecapProgress::default(), } } @@ -147,6 +168,18 @@ impl ThreadEventStore { } fn push_notification_inner(&mut self, notification: Cow<'_, ServerNotification>) { + let user_item = match notification.as_ref() { + ServerNotification::ItemStarted(n) => Some((&n.turn_id, &n.item)), + ServerNotification::ItemCompleted(n) => Some((&n.turn_id, &n.item)), + _ => None, + }; + if let Some((turn_id, ThreadItem::UserMessage { content, .. })) = user_item + && turn_id.len() <= 512 + && crate::chatwidget::realtime_delegation_input(content).is_some() + && !self.delegated_turns.contains(turn_id) + { + self.delegated_turns.push_back(turn_id.clone()); + } self.pending_interactive_replay .note_server_notification(notification.as_ref()); match notification.as_ref() { @@ -204,6 +237,27 @@ impl ThreadEventStore { } self.push_replay_notification(notification); + self.trim_delegated_turns(); + } + + fn trim_delegated_turns(&mut self) { + while self.delegated_turns.len() > self.capacity { + // Keep an evicted user marker while this turn still has buffered output. + let stale = self.delegated_turns.iter().position(|turn_id| { + !self.buffer.iter().any(|event| { + matches!(event, ThreadBufferedEvent::Notification(n) if match n.as_ref() { + ServerNotification::ItemCompleted(item) => &item.turn_id == turn_id, + ServerNotification::AgentMessageDelta(delta) => &delta.turn_id == turn_id, + ServerNotification::ReasoningSummaryTextDelta(delta) => &delta.turn_id == turn_id, + ServerNotification::ReasoningTextDelta(delta) => &delta.turn_id == turn_id, + ServerNotification::ReasoningSummaryPartAdded(part) => &part.turn_id == turn_id, + ServerNotification::TurnCompleted(turn) => &turn.turn.id == turn_id, + _ => false, + }) + }) + }); + self.delegated_turns.remove(stale.unwrap_or(0)); + } } pub(super) fn push_request(&mut self, request: ServerRequest) { @@ -242,6 +296,7 @@ impl ThreadEventStore { pub(super) fn snapshot(&self) -> ThreadEventSnapshot { let mut snapshot = ThreadEventSnapshot { session: self.session.clone(), + delegated_turns: self.delegated_turns.iter().cloned().collect(), turns: self.turns.clone(), // Thread switches replay buffered events into a rebuilt ChatWidget. Only replay // interactive prompts that are still pending, or answered approvals/input will reappear. @@ -263,6 +318,99 @@ impl ThreadEventStore { if let Some(latest_turn_id) = &self.latest_turn_id { replay_filter::omit_resolved_misalignment_errors(&mut snapshot, latest_turn_id); } + let delegated = snapshot + .turns + .iter() + .filter(|turn| { + turn.items.iter().any(|item| { + matches!(item, ThreadItem::UserMessage { content, .. } + if crate::chatwidget::realtime_delegation_input(content).is_some()) + }) + }) + .map(|turn| turn.id.clone()) + .chain(self.delegated_turns.iter().cloned()) + .collect::>(); + for turn in &mut snapshot.turns { + hide_private_items_after_voice_handoff(&mut turn.items, delegated.contains(&turn.id)); + } + let buffered_markers = snapshot + .events + .iter() + .filter_map(|event| match event { + ThreadBufferedEvent::Notification(notification) => match notification.as_ref() { + ServerNotification::ItemStarted(n) if is_voice_handoff_item(&n.item) => { + Some(n.turn_id.clone()) + } + ServerNotification::ItemCompleted(n) if is_voice_handoff_item(&n.item) => { + Some(n.turn_id.clone()) + } + _ => None, + }, + _ => None, + }) + .collect::>(); + let mut voice_started = delegated + .difference(&buffered_markers) + .cloned() + .collect::>(); + let mut typed_items = std::collections::HashSet::new(); + snapshot.events.retain_mut(|event| { + let ThreadBufferedEvent::Notification(notification) = event else { + return true; + }; + match notification.as_ref() { + ServerNotification::ItemStarted(n) if is_voice_handoff_item(&n.item) => { + voice_started.insert(n.turn_id.clone()); + } + ServerNotification::ItemCompleted(n) if is_voice_handoff_item(&n.item) => { + voice_started.insert(n.turn_id.clone()); + } + ServerNotification::ItemStarted(n) => { + if let ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. } = + &n.item + { + let key = (n.turn_id.clone(), id.clone()); + if voice_started.contains(&n.turn_id) { + typed_items.remove(&key); + } else { + typed_items.insert(key); + } + } + } + _ => {} + } + match notification.as_mut() { + ServerNotification::AgentMessageDelta(n) => { + !voice_started.contains(&n.turn_id) + || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone())) + } + ServerNotification::ReasoningSummaryTextDelta(n) => { + !voice_started.contains(&n.turn_id) + || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone())) + } + ServerNotification::ReasoningTextDelta(n) => { + !voice_started.contains(&n.turn_id) + || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone())) + } + ServerNotification::ReasoningSummaryPartAdded(n) => { + !voice_started.contains(&n.turn_id) + || typed_items.contains(&(n.turn_id.clone(), n.item_id.clone())) + } + ServerNotification::ItemCompleted(n) if voice_started.contains(&n.turn_id) => { + matches!(&n.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. } + if typed_items.contains(&(n.turn_id.clone(), id.clone()))) + || !crate::chatwidget::is_private_realtime_agent_item(&n.item) + } + ServerNotification::TurnCompleted(n) if delegated.contains(&n.turn.id) => { + hide_private_items_after_voice_handoff( + &mut n.turn.items, + /*delegated*/ true, + ); + true + } + _ => true, + } + }); snapshot } @@ -430,12 +578,16 @@ mod tests { use codex_app_server_protocol::HookRunSummary as AppServerHookRunSummary; use codex_app_server_protocol::HookScope as AppServerHookScope; use codex_app_server_protocol::HookStartedNotification; + use codex_app_server_protocol::ItemCompletedNotification; + use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::McpToolCallProgressNotification; + use codex_app_server_protocol::ReasoningSummaryTextDeltaNotification; use codex_app_server_protocol::RequestId as AppServerRequestId; use codex_app_server_protocol::ThreadRealtimeAudioChunk; use codex_app_server_protocol::ThreadRealtimeOutputAudioDeltaNotification; use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnStartedNotification; + use codex_app_server_protocol::UserInput; use codex_config::types::ApprovalsReviewer; use codex_protocol::models::PermissionProfile; use pretty_assertions::assert_eq; @@ -818,4 +970,93 @@ mod tests { serde_json::to_value(notification).expect("MCP notification should serialize"), ); } + + #[test] + fn buffered_voice_handoff_retains_only_preexisting_typed_reasoning() { + let thread_id = ThreadId::new().to_string(); + let turn_id = "mixed-reasoning-turn".to_string(); + let mut store = ThreadEventStore::new(/*capacity*/ 16); + let typed = ThreadItem::Reasoning { + id: "typed-reasoning".into(), + summary: Vec::new(), + content: Vec::new(), + }; + store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item: typed, + started_at_ms: 0, + })); + store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item: ThreadItem::UserMessage { + id: "voice-marker".into(), + client_id: None, + content: vec![UserInput::Text { + text: + "spoken follow-up" + .into(), + text_elements: Vec::new(), + }], + }, + started_at_ms: 0, + })); + for (id, text) in [ + ("typed-reasoning", "Typed tail"), + ("private-reasoning", "Private voice reasoning"), + ] { + if id == "private-reasoning" { + store.push_notification(ServerNotification::ItemStarted(ItemStartedNotification { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item: ThreadItem::Reasoning { + id: id.into(), + summary: Vec::new(), + content: Vec::new(), + }, + started_at_ms: 0, + })); + } + store.push_notification(ServerNotification::ReasoningSummaryTextDelta( + ReasoningSummaryTextDeltaNotification { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item_id: id.into(), + delta: text.into(), + summary_index: 0, + }, + )); + store.push_notification(ServerNotification::ItemCompleted( + ItemCompletedNotification { + thread_id: thread_id.clone(), + turn_id: turn_id.clone(), + item: ThreadItem::Reasoning { + id: id.into(), + summary: vec![text.into()], + content: Vec::new(), + }, + completed_at_ms: 0, + }, + )); + } + + let retained = store + .snapshot() + .events + .into_iter() + .filter_map(|event| match event { + ThreadBufferedEvent::Notification(notification) => match *notification { + ServerNotification::ReasoningSummaryTextDelta(n) => Some(n.delta), + ServerNotification::ItemCompleted(n) => match n.item { + ThreadItem::Reasoning { summary, .. } => Some(summary.join("")), + _ => None, + }, + _ => None, + }, + _ => None, + }) + .collect::>(); + assert_eq!(retained, vec!["Typed tail", "Typed tail"]); + } } diff --git a/codex-rs/tui/src/app/thread_routing.rs b/codex-rs/tui/src/app/thread_routing.rs index e688317558..45b370a8bd 100644 --- a/codex-rs/tui/src/app/thread_routing.rs +++ b/codex-rs/tui/src/app/thread_routing.rs @@ -13,8 +13,32 @@ use codex_app_server_protocol::TurnInterruptParams; use codex_app_server_protocol::TurnInterruptResponse; use codex_app_server_protocol::WarningNotification; +// Leave time for side-thread cleanup and unsubscribe inside the two-second exit budget. +const REALTIME_STOP_TIMEOUT: Duration = Duration::from_secs(/*secs*/ 1); + impl App { + pub(super) async fn stop_realtime_conversation(&mut self, app_server: &mut AppServerSession) { + let Some(thread_id) = self.chat_widget.reset_realtime_conversation() else { + return; + }; + match tokio::time::timeout( + REALTIME_STOP_TIMEOUT, + app_server.thread_realtime_stop(thread_id), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + tracing::warn!(%thread_id, %error, "failed to stop voice conversation"); + } + Err(_) => { + tracing::warn!(%thread_id, "timed out stopping voice conversation"); + } + } + } + pub(super) async fn shutdown_current_thread(&mut self, app_server: &mut AppServerSession) { + self.stop_realtime_conversation(app_server).await; self.shutdown_side_threads(app_server).await; if let Some(thread_id) = self.chat_widget.thread_id() { if let Err(err) = app_server.thread_unsubscribe(thread_id).await { @@ -446,6 +470,15 @@ impl App { op: AppCommand, ) -> Result<()> { let Some(thread_id) = self.active_thread_id else { + if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op { + self.chat_widget + .restore_undelivered_realtime_speech(*delivery_id); + } + if let AppCommand::RealtimeConversationStart { thread_id, .. } = &op + && self.chat_widget.thread_id() == Some(*thread_id) + { + self.chat_widget.reset_realtime_conversation(); + } self.chat_widget .add_error_message("No active thread is available.".to_string()); return Ok(()); @@ -461,12 +494,25 @@ impl App { op: AppCommand, ) -> Result<()> { if self.thread_unavailable(thread_id) { + if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op { + self.chat_widget + .restore_undelivered_realtime_speech(*delivery_id); + } + if let AppCommand::RealtimeConversationStart { thread_id, .. } = &op + && self.chat_widget.thread_id() == Some(*thread_id) + { + self.chat_widget.reset_realtime_conversation(); + } self.chat_widget.add_error_message( "This conversation is read-only or unavailable; no operation was sent.".into(), ); return Ok(()); } if self.chat_widget.rejects_misalignment_policy_op(&op) { + if let AppCommand::RealtimeConversationSpeech { delivery_id, .. } = &op { + self.chat_widget + .restore_undelivered_realtime_speech(*delivery_id); + } return Ok(()); } @@ -867,6 +913,60 @@ impl App { .await?; Ok(true) } + AppCommand::RealtimeConversationStart { + thread_id: realtime_thread_id, + offer_sdp, + } => { + if *realtime_thread_id != thread_id { + return Ok(true); + } + let model = self + .chat_widget + .config_ref() + .experimental_realtime_ws_model + .clone(); + app_server + .thread_realtime_start( + *realtime_thread_id, + String::from(offer_sdp.clone()), + model, + ) + .await?; + Ok(true) + } + AppCommand::RealtimeConversationStop { + thread_id: realtime_thread_id, + } => { + app_server.thread_realtime_stop(*realtime_thread_id).await?; + Ok(true) + } + AppCommand::RealtimeConversationSpeech { + thread_id: realtime_thread_id, + attempt_id, + input_generation, + delivery_id, + text, + } => { + if !self.chat_widget.has_pending_realtime_speech(*delivery_id) { + return Ok(true); + } + if *realtime_thread_id != thread_id + || !self.chat_widget.is_current_realtime_attempt( + *realtime_thread_id, + *attempt_id, + *input_generation, + ) + { + self.chat_widget + .restore_undelivered_realtime_speech(*delivery_id); + return Ok(true); + } + app_server + .thread_realtime_append_speech(*realtime_thread_id, text.as_str().to_owned()) + .await?; + self.chat_widget.accept_realtime_speech(*delivery_id); + Ok(true) + } AppCommand::RunUserShellCommand { command } => { app_server .thread_shell_command(thread_id, command.to_string()) @@ -1136,6 +1236,7 @@ impl App { guard.push_notification_ref(¬ification); Some(notification) } else { + self.retain_inactive_realtime_transcript(thread_id, ¬ification); guard.push_notification(notification); None }; @@ -1423,6 +1524,8 @@ impl App { } } let should_buffer_initial_replay = !turns.is_empty(); + let replayed_final_items = realtime_delivery::completed_agent_items_from_turns(&turns); + let replayed_voice_texts = realtime_delivery::replayed_voice_texts_from_turns(&turns); if should_buffer_initial_replay { self.app_event_tx .send(AppEvent::BeginInitialHistoryReplayBuffer); @@ -1438,6 +1541,10 @@ impl App { self.app_event_tx .send(AppEvent::EndInitialHistoryReplayBuffer); } + self.restore_realtime_replay_state_after_replay( + &replayed_final_items, + replayed_voice_texts, + ); if matches!(presentation, ThreadAttachPresentation::PromptEdit) { self.chat_widget.emit_prompt_edit_thread_event(); } @@ -1651,6 +1758,8 @@ impl App { mut snapshot: ThreadEventSnapshot, resume_restored_queue: bool, ) { + let replayed_final_items = realtime_delivery::completed_agent_items(&snapshot); + let replayed_voice_texts = realtime_delivery::replayed_voice_texts(&snapshot); replay_filter::omit_completed_agent_deltas(&mut snapshot.events); let request_changes = snapshot .events @@ -1694,6 +1803,10 @@ impl App { self.chat_widget.handle_thread_session(session); } } + for turn_id in &snapshot.delegated_turns { + self.chat_widget + .remember_realtime_delegated_reasoning_turn(turn_id); + } let recovered_input = snapshot .input_state .as_ref() @@ -1728,6 +1841,10 @@ impl App { if recovered_input.is_some() { self.chat_widget.restore_reconnected_input(recovered_input); } + self.restore_realtime_replay_state_after_replay( + &replayed_final_items, + replayed_voice_texts, + ); self.chat_widget .set_queue_autosend_suppressed(/*suppressed*/ false); self.chat_widget diff --git a/codex-rs/tui/src/app_command.rs b/codex-rs/tui/src/app_command.rs index f54b00497d..82bc948942 100644 --- a/codex-rs/tui/src/app_command.rs +++ b/codex-rs/tui/src/app_command.rs @@ -1,3 +1,4 @@ +use std::fmt; use std::path::PathBuf; use codex_app_server_protocol::AskForApproval; @@ -9,6 +10,7 @@ use codex_app_server_protocol::ReviewTarget; use codex_app_server_protocol::ToolRequestUserInputResponse; use codex_app_server_protocol::UserInput; use codex_config::types::ApprovalsReviewer; +use codex_protocol::ThreadId; use codex_protocol::approvals::GuardianAssessmentEvent; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::Personality; @@ -19,13 +21,98 @@ use codex_protocol::models::PermissionProfile; use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig; use codex_protocol::request_permissions::RequestPermissionsResponse; use serde::Serialize; +use serde::Serializer; use serde_json::Value; +/// Keeps ICE credentials out of command diagnostics and session recordings. +/// Convert back to a string only when sending the offer to app-server signaling. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RealtimeOfferSdp(String); + +impl From for RealtimeOfferSdp { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From for String { + fn from(value: RealtimeOfferSdp) -> Self { + value.0 + } +} + +impl fmt::Debug for RealtimeOfferSdp { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("") + } +} + +impl Serialize for RealtimeOfferSdp { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str("") + } +} + +/// Keeps spoken answers out of command diagnostics and session recordings. +/// Convert back to a string only at the app-server signaling boundary. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct RealtimeSpeechText(String); + +impl From for RealtimeSpeechText { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From for String { + fn from(value: RealtimeSpeechText) -> Self { + value.0 + } +} + +impl RealtimeSpeechText { + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for RealtimeSpeechText { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("") + } +} + +impl Serialize for RealtimeSpeechText { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str("") + } +} + #[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, PartialEq, Serialize)] pub(crate) enum AppCommand { Interrupt, CleanBackgroundTerminals, + RealtimeConversationStart { + thread_id: ThreadId, + offer_sdp: RealtimeOfferSdp, + }, + RealtimeConversationStop { + thread_id: ThreadId, + }, + RealtimeConversationSpeech { + thread_id: ThreadId, + attempt_id: u64, + input_generation: u64, + delivery_id: u64, + text: RealtimeSpeechText, + }, RunUserShellCommand { command: String, }, @@ -252,3 +339,7 @@ impl From<&AppCommand> for AppCommand { value.clone() } } + +#[cfg(test)] +#[path = "app_command_tests.rs"] +mod tests; diff --git a/codex-rs/tui/src/app_command_tests.rs b/codex-rs/tui/src/app_command_tests.rs new file mode 100644 index 0000000000..c765205c62 --- /dev/null +++ b/codex-rs/tui/src/app_command_tests.rs @@ -0,0 +1,70 @@ +//! Protect connection credentials in diagnostics without changing signaling data. + +use super::AppCommand; +use codex_protocol::ThreadId; +use pretty_assertions::assert_eq; +use serde_json::json; + +#[test] +fn realtime_offer_is_redacted_in_diagnostics_and_preserved_for_signaling() { + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000001").expect("valid thread id"); + let original_offer = + "v=0\r\na=ice-ufrag:private-user\r\na=ice-pwd:private-credential\r\n".to_string(); + let command = AppCommand::RealtimeConversationStart { + thread_id, + offer_sdp: original_offer.clone().into(), + }; + + assert_eq!( + format!("{command:?}"), + format!("RealtimeConversationStart {{ thread_id: {thread_id:?}, offer_sdp: }}") + ); + assert_eq!( + serde_json::to_value(&command).expect("command diagnostics serialize"), + json!({ + "RealtimeConversationStart": { + "thread_id": thread_id, + "offer_sdp": "" + } + }) + ); + + let AppCommand::RealtimeConversationStart { offer_sdp, .. } = command else { + panic!("expected the start command"); + }; + assert_eq!(String::from(offer_sdp), original_offer); +} + +#[test] +fn realtime_speech_is_redacted_in_diagnostics_and_preserved_for_signaling() { + let thread_id = + ThreadId::from_string("00000000-0000-0000-0000-000000000001").expect("valid thread id"); + let original_text = "Private spoken answer".to_string(); + let command = AppCommand::RealtimeConversationSpeech { + thread_id, + attempt_id: 1, + input_generation: 2, + delivery_id: 3, + text: original_text.clone().into(), + }; + + assert!(!format!("{command:?}").contains(&original_text)); + assert_eq!( + serde_json::to_value(&command).expect("command diagnostics serialize"), + json!({ + "RealtimeConversationSpeech": { + "thread_id": thread_id, + "attempt_id": 1, + "input_generation": 2, + "delivery_id": 3, + "text": "" + } + }) + ); + + let AppCommand::RealtimeConversationSpeech { text, .. } = command else { + panic!("expected the speech command"); + }; + assert_eq!(String::from(text), original_text); +} diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 1df5a1a7f4..285b7de6d7 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -63,6 +63,7 @@ use codex_plugin::PluginCapabilitySummary; use codex_protocol::config_types::CollaborationModeMask; use codex_protocol::config_types::Personality; use codex_protocol::models::ActivePermissionProfile; +use codex_realtime_webrtc::StartedRealtimeWebrtcSession; use crate::history_cell::HistoryCell; @@ -1053,6 +1054,25 @@ pub(crate) enum AppEvent { /// Update the current personality in the running app and widget. UpdatePersonality(Personality), + /// Result of creating a TUI-owned WebRTC offer for an active thread. + RealtimeWebrtcOfferCreated { + thread_id: ThreadId, + attempt_id: u64, + result: Result, + }, + + /// Result of establishing the WebRTC connection for an active voice attempt. + RealtimeWebrtcConnected { + thread_id: ThreadId, + attempt_id: u64, + result: Result<(), codex_realtime_webrtc::ConnectionError>, + }, + + /// Stop voice on its original thread after its chat widget is replaced. + StopRealtimeConversation { + thread_id: ThreadId, + }, + /// Finish a settings selection after its preceding update events have been applied. SettingsSelectionClosed, /// Run after any nested settings events emitted while handling the close event. diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index 9583524099..c5b74996e8 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -6,6 +6,7 @@ mod fs; mod history; mod models; +mod realtime; mod rollout_history; mod thread_list; diff --git a/codex-rs/tui/src/app_server_session/realtime.rs b/codex-rs/tui/src/app_server_session/realtime.rs new file mode 100644 index 0000000000..f9fff2c00b --- /dev/null +++ b/codex-rs/tui/src/app_server_session/realtime.rs @@ -0,0 +1,91 @@ +//! App-server signaling for TUI-owned realtime WebRTC sessions. + +use super::AppServerSession; +use codex_app_server_protocol::ClientRequest; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechParams; +use codex_app_server_protocol::ThreadRealtimeAppendSpeechResponse; +use codex_app_server_protocol::ThreadRealtimeStartParams; +use codex_app_server_protocol::ThreadRealtimeStartResponse; +use codex_app_server_protocol::ThreadRealtimeStartTransport; +use codex_app_server_protocol::ThreadRealtimeStopParams; +use codex_app_server_protocol::ThreadRealtimeStopResponse; +use codex_protocol::ThreadId; +use codex_protocol::protocol::RealtimeConversationVersion; +use codex_protocol::protocol::RealtimeOutputModality; +use color_eyre::eyre::Result; +use color_eyre::eyre::WrapErr; + +impl AppServerSession { + pub(crate) async fn thread_realtime_start( + &mut self, + thread_id: ThreadId, + offer_sdp: String, + model: Option, + ) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadRealtimeStartResponse = self + .client + .request_typed(ClientRequest::ThreadRealtimeStart { + request_id, + params: ThreadRealtimeStartParams { + thread_id: thread_id.to_string(), + client_managed_handoffs: Some(true), + delegation_ack_filler: None, + flush_transcript_tail_on_session_end: None, + codex_responses_as_items: None, + codex_response_item_prefix: None, + codex_response_handoff_mode: None, + codex_response_handoff_channel_prefixes: None, + model, + output_modality: RealtimeOutputModality::Audio, + include_startup_context: Some(false), + initial_items: None, + realtime_start_instructions: None, + realtime_end_instructions: None, + prompt: None, + realtime_session_id: None, + transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp }), + version: Some(RealtimeConversationVersion::V3), + voice: None, + }, + }) + .await + .wrap_err("thread/realtime/start failed in TUI")?; + Ok(()) + } + + pub(crate) async fn thread_realtime_stop(&mut self, thread_id: ThreadId) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadRealtimeStopResponse = self + .client + .request_typed(ClientRequest::ThreadRealtimeStop { + request_id, + params: ThreadRealtimeStopParams { + thread_id: thread_id.to_string(), + }, + }) + .await + .wrap_err("thread/realtime/stop failed in TUI")?; + Ok(()) + } + + pub(crate) async fn thread_realtime_append_speech( + &mut self, + thread_id: ThreadId, + text: String, + ) -> Result<()> { + let request_id = self.next_request_id(); + let _: ThreadRealtimeAppendSpeechResponse = self + .client + .request_typed(ClientRequest::ThreadRealtimeAppendSpeech { + request_id, + params: ThreadRealtimeAppendSpeechParams { + thread_id: thread_id.to_string(), + text, + }, + }) + .await + .wrap_err("thread/realtime/appendSpeech failed in TUI")?; + Ok(()) + } +} diff --git a/codex-rs/tui/src/bottom_pane/chat_composer.rs b/codex-rs/tui/src/bottom_pane/chat_composer.rs index 967c712fe9..6d084e067f 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer.rs @@ -554,6 +554,7 @@ pub(crate) struct ChatComposer { mentions_v2_enabled: bool, goal_command_enabled: bool, personality_command_enabled: bool, + voice_command_enabled: bool, worktrees_enabled: bool, windows_degraded_sandbox_active: bool, side_conversation_active: bool, @@ -718,6 +719,7 @@ impl ChatComposer { mentions_v2_enabled: false, goal_command_enabled: false, personality_command_enabled: false, + voice_command_enabled: false, worktrees_enabled: false, windows_degraded_sandbox_active: false, side_conversation_active: false, @@ -924,6 +926,10 @@ impl ChatComposer { self.goal_command_enabled = enabled; } + pub fn set_voice_command_enabled(&mut self, enabled: bool) { + self.voice_command_enabled = enabled; + } + /// Replace composer, editor, and footer-hint key bindings from one runtime snapshot. /// /// Submit and queue bindings are cached here because composer dispatch must @@ -1035,19 +1041,11 @@ impl ChatComposer { .unwrap_or_else(|| footer_height(&footer_props)); let footer_spacing = Self::footer_spacing(footer_hint_height); let footer_total_height = footer_hint_height + footer_spacing; - let popup_constraint = match &self.popups.active { - ActivePopup::Command(popup) => { - Constraint::Max(popup.calculate_required_height(area.width)) - } - ActivePopup::File(popup) => Constraint::Max(popup.calculate_required_height()), - ActivePopup::Skill(popup) => { - Constraint::Max(popup.calculate_required_height(area.width)) - } - ActivePopup::MentionV2(popup) => { - Constraint::Max(popup.calculate_required_height(area.width)) - } - ActivePopup::None => Constraint::Max(footer_total_height), - }; + let popup_constraint = Constraint::Max( + self.popups + .active + .required_height(area.width, footer_total_height), + ); let [composer_rect, popup_rect] = Layout::vertical([Constraint::Min(3), popup_constraint]).areas(area); let mut textarea_rect = composer_rect.inset(Insets::tlbr( @@ -4656,13 +4654,10 @@ impl ChatComposer { + remote_images_height + remote_images_separator + 2 - + match &self.popups.active { - ActivePopup::None => footer_total_height, - ActivePopup::Command(c) => c.calculate_required_height(width), - ActivePopup::File(c) => c.calculate_required_height(), - ActivePopup::Skill(c) => c.calculate_required_height(width), - ActivePopup::MentionV2(c) => c.calculate_required_height(width), - } + + self + .popups + .active + .required_height(width, footer_total_height) } } diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs b/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs index 2f97ecda52..da2dfd908b 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer/popup_state.rs @@ -112,6 +112,18 @@ pub(super) enum ActivePopup { MentionV2(MentionV2Popup), } +impl ActivePopup { + pub(super) fn required_height(&self, width: u16, footer_total_height: u16) -> u16 { + match self { + Self::None => footer_total_height, + Self::Command(popup) => popup.calculate_required_height(width), + Self::File(popup) => popup.calculate_required_height(), + Self::Skill(popup) => popup.calculate_required_height(width), + Self::MentionV2(popup) => popup.calculate_required_height(width), + } + } +} + #[cfg(test)] #[path = "popup_state_tests.rs"] mod tests; diff --git a/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs index 22fce854f4..8e4ff2f87b 100644 --- a/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs +++ b/codex-rs/tui/src/bottom_pane/chat_composer/slash_input.rs @@ -178,6 +178,7 @@ impl<'a> SlashInput<'a> { service_tier_commands_enabled: self.command_flags.service_tier_commands_enabled, goal_command_enabled: self.command_flags.goal_command_enabled, personality_command_enabled: self.command_flags.personality_command_enabled, + voice_command_enabled: self.command_flags.voice_command_enabled, worktrees_enabled: self.command_flags.worktrees_enabled, windows_degraded_sandbox_active: self.command_flags.allow_elevate_sandbox, side_conversation_active: self.command_flags.side_conversation_active, @@ -216,6 +217,7 @@ impl ChatComposer { service_tier_commands_enabled: self.service_tier_commands_enabled, goal_command_enabled: self.goal_command_enabled, personality_command_enabled: self.personality_command_enabled, + voice_command_enabled: self.voice_command_enabled, worktrees_enabled: self.worktrees_enabled, allow_elevate_sandbox: self.windows_degraded_sandbox_active, side_conversation_active: self.side_conversation_active, diff --git a/codex-rs/tui/src/bottom_pane/command_popup.rs b/codex-rs/tui/src/bottom_pane/command_popup.rs index 81cc8013d8..f577205eba 100644 --- a/codex-rs/tui/src/bottom_pane/command_popup.rs +++ b/codex-rs/tui/src/bottom_pane/command_popup.rs @@ -48,6 +48,7 @@ pub(crate) struct CommandPopupFlags { pub(crate) service_tier_commands_enabled: bool, pub(crate) goal_command_enabled: bool, pub(crate) personality_command_enabled: bool, + pub(crate) voice_command_enabled: bool, pub(crate) worktrees_enabled: bool, pub(crate) windows_degraded_sandbox_active: bool, pub(crate) side_conversation_active: bool, @@ -63,6 +64,7 @@ impl From for BuiltinCommandFlags { service_tier_commands_enabled: value.service_tier_commands_enabled, goal_command_enabled: value.goal_command_enabled, personality_command_enabled: value.personality_command_enabled, + voice_command_enabled: value.voice_command_enabled, worktrees_enabled: value.worktrees_enabled, allow_elevate_sandbox: value.windows_degraded_sandbox_active, side_conversation_active: value.side_conversation_active, @@ -412,6 +414,31 @@ mod tests { insta::assert_snapshot!("command_popup_app", format!("{buf:?}")); } + #[cfg(target_os = "macos")] + #[test] + fn voice_command_popup_snapshot() { + let mut popup = CommandPopup::new( + CommandPopupFlags { + voice_command_enabled: true, + ..CommandPopupFlags::default() + }, + Vec::new(), + ); + popup.on_composer_text_change("/voice".to_string()); + + let width = 72; + let area = Rect::new( + /*x*/ 0, + /*y*/ 0, + width, + popup.calculate_required_height(width), + ); + let mut buf = Buffer::empty(area); + popup.render_ref(area, &mut buf); + + insta::assert_snapshot!("command_popup_voice", format!("{buf:?}")); + } + #[cfg(target_os = "macos")] #[test] fn default_command_popup_items_snapshot() { @@ -537,6 +564,7 @@ mod tests { service_tier_commands_enabled: false, goal_command_enabled: false, personality_command_enabled: true, + voice_command_enabled: false, worktrees_enabled: true, windows_degraded_sandbox_active: false, side_conversation_active: false, @@ -565,6 +593,7 @@ mod tests { service_tier_commands_enabled: false, goal_command_enabled: false, personality_command_enabled: false, + voice_command_enabled: false, worktrees_enabled: false, windows_degraded_sandbox_active: false, side_conversation_active: false, @@ -598,6 +627,7 @@ mod tests { service_tier_commands_enabled: false, goal_command_enabled: false, personality_command_enabled: true, + voice_command_enabled: false, worktrees_enabled: true, windows_degraded_sandbox_active: false, side_conversation_active: false, diff --git a/codex-rs/tui/src/bottom_pane/footer.rs b/codex-rs/tui/src/bottom_pane/footer.rs index 997866be7a..d5cbd09067 100644 --- a/codex-rs/tui/src/bottom_pane/footer.rs +++ b/codex-rs/tui/src/bottom_pane/footer.rs @@ -875,9 +875,17 @@ pub(crate) fn footer_hint_items_line(items: &[(String, String)]) -> Line<'static for (idx, (key, label)) in items.iter().enumerate() { spans.push(" ".into()); spans.push(key.clone().bold()); - spans.push(format!(" {label}").into()); + if idx == 0 + && key == "voice" + && let Some(label) = label.strip_prefix("● ") + { + spans.push(" ●".red()); + spans.push(format!(" {label}").into()); + } else { + spans.push(format!(" {label}").into()); + } if idx + 1 != items.len() { - spans.push(" ".into()); + spans.push((if items[0].0 == "voice" { " " } else { " " }).into()); } } Line::from(spans) @@ -1313,6 +1321,43 @@ mod tests { use ratatui::backend::Backend; use ratatui::backend::TestBackend; + #[test] + fn voice_live_microphone_indicator_is_red() { + let line = footer_hint_items_line(&[("voice".into(), "● listen".into())]); + assert_eq!(line.spans[2].style.fg, Some(ratatui::style::Color::Red)); + } + + #[test] + fn voice_footer_rendering_preserves_text_and_styles() { + let items = [ + ("voice".into(), "● listen".into()), + ("ctrl+m".into(), "mute".into()), + ]; + let mut terminal = + Terminal::new(TestBackend::new(/*width*/ 40, /*height*/ 1)).expect("create terminal"); + terminal + .draw(|frame| render_footer_hint_items(frame.area(), frame.buffer_mut(), &items)) + .expect("render voice footer"); + let backend = terminal.backend(); + let mut previous_style = None; + let style_runs = (0..40) + .filter_map(|x| { + let cell = backend.buffer().cell((x, 0))?; + let style = cell.style(); + if previous_style == Some(style) { + None + } else { + previous_style = Some(style); + Some((x, style)) + } + }) + .collect::>(); + insta::assert_debug_snapshot!( + "voice_footer_rendered_styles", + (backend.to_string(), style_runs) + ); + } + fn snapshot_footer(name: &str, props: FooterProps) { snapshot_footer_with_mode_indicator( name, /*width*/ 80, &props, /*collaboration_mode_indicator*/ None, diff --git a/codex-rs/tui/src/bottom_pane/mod.rs b/codex-rs/tui/src/bottom_pane/mod.rs index f249fad8b2..00712baab9 100644 --- a/codex-rs/tui/src/bottom_pane/mod.rs +++ b/codex-rs/tui/src/bottom_pane/mod.rs @@ -546,6 +546,11 @@ impl BottomPane { self.request_redraw(); } + pub fn set_voice_command_enabled(&mut self, enabled: bool) { + self.composer.set_voice_command_enabled(enabled); + self.request_redraw(); + } + pub(crate) fn set_side_conversation_active(&mut self, active: bool) { self.composer.set_side_conversation_active(active); self.request_redraw(); diff --git a/codex-rs/tui/src/bottom_pane/slash_commands.rs b/codex-rs/tui/src/bottom_pane/slash_commands.rs index 61393648e7..8c478b3e31 100644 --- a/codex-rs/tui/src/bottom_pane/slash_commands.rs +++ b/codex-rs/tui/src/bottom_pane/slash_commands.rs @@ -62,6 +62,7 @@ pub(crate) struct BuiltinCommandFlags { pub(crate) service_tier_commands_enabled: bool, pub(crate) goal_command_enabled: bool, pub(crate) personality_command_enabled: bool, + pub(crate) voice_command_enabled: bool, pub(crate) worktrees_enabled: bool, pub(crate) allow_elevate_sandbox: bool, pub(crate) side_conversation_active: bool, @@ -79,6 +80,7 @@ pub(crate) fn builtins_for_input(flags: BuiltinCommandFlags) -> Vec<(&'static st .filter(|(_, cmd)| flags.goal_command_enabled || *cmd != SlashCommand::Goal) .filter(|(_, cmd)| flags.worktrees_enabled || *cmd != SlashCommand::Worktree) .filter(|(_, cmd)| flags.personality_command_enabled || *cmd != SlashCommand::Personality) + .filter(|(_, cmd)| flags.voice_command_enabled || *cmd != SlashCommand::Voice) .filter(|(_, cmd)| !flags.side_conversation_active || cmd.available_in_side_conversation()) .collect() } @@ -173,6 +175,7 @@ mod tests { service_tier_commands_enabled: true, goal_command_enabled: true, personality_command_enabled: true, + voice_command_enabled: true, worktrees_enabled: true, allow_elevate_sandbox: true, side_conversation_active: false, @@ -282,6 +285,13 @@ mod tests { assert_eq!(find_builtin_command("goal", flags), None); } + #[test] + fn voice_command_is_hidden_when_disabled() { + let mut flags = all_enabled_flags(); + flags.voice_command_enabled = false; + assert_eq!(find_builtin_command("voice", flags), None); + } + #[test] fn usage_command_is_hidden_from_input_when_account_token_activity_is_disabled() { let mut flags = all_enabled_flags(); diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap new file mode 100644 index 0000000000..417c3b62eb --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__command_popup__tests__command_popup_voice.snap @@ -0,0 +1,15 @@ +--- +source: tui/src/bottom_pane/command_popup.rs +expression: "format!(\"{buf:?}\")" +--- +Buffer { + area: Rect { x: 0, y: 0, width: 72, height: 1 }, + content: [ + " /voice start or stop a live voice conversation ", + ], + styles: [ + x: 0, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + x: 2, y: 0, fg: Cyan, bg: Reset, underline: Reset, modifier: BOLD, + x: 49, y: 0, fg: Reset, bg: Reset, underline: Reset, modifier: NONE, + ] +} diff --git a/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap new file mode 100644 index 0000000000..d71648f1d6 --- /dev/null +++ b/codex-rs/tui/src/bottom_pane/snapshots/codex_tui__bottom_pane__footer__tests__voice_footer_rendered_styles.snap @@ -0,0 +1,34 @@ +--- +source: tui/src/bottom_pane/footer.rs +assertion_line: 1355 +expression: "(backend.to_string(), style_runs)" +--- +( + "\" voice ● listen ctrl+m mute \"\n", + [ + ( + 0, + Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset), + ), + ( + 3, + Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset).bold(), + ), + ( + 8, + Style::new().red().bg(Color::Reset).underline_color(Color::Reset), + ), + ( + 10, + Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset), + ), + ( + 20, + Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset).bold(), + ), + ( + 26, + Style::new().fg(Color::Reset).bg(Color::Reset).underline_color(Color::Reset), + ), + ], +) diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index afaa3fc953..628e49b352 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -357,6 +357,7 @@ mod mcp_startup; use self::mcp_startup::McpStartupStatus; mod misalignment_policy; pub(crate) use misalignment_policy::MisalignmentReview; +pub(crate) use misalignment_policy::MisalignmentTurnSource; mod pets; mod session_flow; mod session_header; @@ -406,7 +407,17 @@ mod recap; mod reset_credits; pub(crate) use self::rate_limits::limit_label_for_window; mod completion; +mod realtime; +pub(crate) use realtime::MAX_REPLAY_TRANSCRIPT_CELLS; +pub(crate) use realtime::MAX_TRANSCRIPT_BYTES; +pub(crate) use realtime::RealtimeTranscriptRecord; +pub(crate) use realtime::is_private_realtime_agent_item; +pub(crate) use realtime::realtime_delegation_display_text; +pub(crate) use realtime::realtime_delegation_input; +#[cfg(test)] +pub(crate) use realtime::tests::activate_voice_for_thread; mod reasoning_shortcuts; +use self::realtime::RealtimeConversationUiState; mod rendering; mod replay; mod review; @@ -631,6 +642,8 @@ pub(crate) struct ChatWidget { last_unified_wait: Option, unified_exec_wait_streak: Option, turn_lifecycle: TurnLifecycleState, + realtime_conversation: RealtimeConversationUiState, + realtime_conversation_available_for_thread: bool, safety_buffering: SafetyBufferingState, task_complete_pending: bool, unified_exec_processes: Vec, @@ -1179,6 +1192,8 @@ impl ChatWidget { self.update_due_hook_visibility(); self.schedule_hook_timer_if_needed(); self.bottom_pane.pre_draw_tick(); + self.flush_realtime_transcript_history(); + self.refresh_realtime_microphone_level(); if let Some(pet) = self.ambient_pet.as_ref() { pet.schedule_next_frame(); } @@ -1295,7 +1310,20 @@ impl ChatWidget { items: &[UserInput], client_id: Option<&str>, from_replay: bool, + turn_id: &str, ) { + if let Some(input) = realtime::realtime_delegation_input(items) { + if !from_replay && self.should_hide_realtime_delegation(turn_id) { + return; + } + let input = realtime::realtime_delegation_display_text(input); + let projected = [UserInput::Text { + text: input, + text_elements: Vec::new(), + }]; + self.on_committed_user_message(&projected, client_id, from_replay, turn_id); + return; + } let display = Self::user_message_display_from_inputs(items); if from_replay { if self.review.is_review_mode { @@ -1920,9 +1948,14 @@ impl ChatWidget { /// the main viewport updates. pub(crate) fn active_cell_transcript_key(&self) -> Option { let cell = self.transcript.active_cell.as_ref(); + let realtime_cell = self.realtime_conversation.live_transcript_cell.as_ref(); let token_activity_cell = self.pending_token_activity_output(); let rate_limit_reset_hint = self.pending_rate_limit_reset_hint(); - if cell.is_none() && token_activity_cell.is_none() && rate_limit_reset_hint.is_none() { + if cell.is_none() + && realtime_cell.is_none() + && token_activity_cell.is_none() + && rate_limit_reset_hint.is_none() + { return None; } Some(ActiveCellTranscriptKey { @@ -1948,6 +1981,13 @@ impl ChatWidget { if let Some(cell) = self.transcript.active_cell.as_ref() { lines.extend(cell.transcript_hyperlink_lines(width)); } + if let Some(cell) = self.realtime_conversation.live_transcript_cell.as_ref() { + let realtime_lines = cell.transcript_hyperlink_lines(width); + if !realtime_lines.is_empty() && !lines.is_empty() { + lines.push(HyperlinkLine::from("")); + } + lines.extend(realtime_lines); + } if let Some(token_activity_cell) = self.pending_token_activity_output() { let token_activity_lines = token_activity_cell.transcript_hyperlink_lines(width); if !token_activity_lines.is_empty() && !lines.is_empty() { @@ -1998,6 +2038,13 @@ fn has_websocket_timing_metrics(summary: RuntimeMetricsSummary) -> bool { impl Drop for ChatWidget { fn drop(&mut self) { + if self.realtime_conversation.handle.is_some() + && let Some(thread_id) = self.thread_id + { + self.app_event_tx + .send(AppEvent::StopRealtimeConversation { thread_id }); + } + self.reset_realtime_conversation(); self.stop_rate_limit_poller(); } } diff --git a/codex-rs/tui/src/chatwidget/constructor.rs b/codex-rs/tui/src/chatwidget/constructor.rs index f45ae82fef..7be8282a2e 100644 --- a/codex-rs/tui/src/chatwidget/constructor.rs +++ b/codex-rs/tui/src/chatwidget/constructor.rs @@ -175,6 +175,8 @@ impl ChatWidget { last_unified_wait: None, unified_exec_wait_streak: None, turn_lifecycle: TurnLifecycleState::new(prevent_idle_sleep), + realtime_conversation: RealtimeConversationUiState::default(), + realtime_conversation_available_for_thread: false, safety_buffering: SafetyBufferingState::default(), task_complete_pending: false, unified_exec_processes: Vec::new(), @@ -291,6 +293,9 @@ impl ChatWidget { widget.sync_worktrees_enabled(); widget.sync_plugins_command_enabled(); widget.sync_goal_command_enabled(); + widget + .bottom_pane + .set_voice_command_enabled(/*enabled*/ false); widget.sync_mentions_v2_enabled(); widget .bottom_pane diff --git a/codex-rs/tui/src/chatwidget/input_submission.rs b/codex-rs/tui/src/chatwidget/input_submission.rs index 2abcb5655d..b0ba5c7d1e 100644 --- a/codex-rs/tui/src/chatwidget/input_submission.rs +++ b/codex-rs/tui/src/chatwidget/input_submission.rs @@ -410,6 +410,7 @@ impl ChatWidget { return (false, None); } self.dismiss_backend_banner_for_new_turn(); + self.note_realtime_typed_input(&submitted_message.text); if render_in_history { self.input_queue.user_turn_pending_start = true; } diff --git a/codex-rs/tui/src/chatwidget/misalignment_policy.rs b/codex-rs/tui/src/chatwidget/misalignment_policy.rs index fcba6416b9..1be4ed3307 100644 --- a/codex-rs/tui/src/chatwidget/misalignment_policy.rs +++ b/codex-rs/tui/src/chatwidget/misalignment_policy.rs @@ -26,6 +26,13 @@ impl MisalignmentReview { pub(super) struct MisalignmentViolation { turn_id: Option, review: Option>, + retired_voice: bool, +} + +#[derive(Clone, Copy, Eq, PartialEq)] +pub(crate) enum MisalignmentTurnSource { + ServerNotification, + AcknowledgedContinuation, } const MISALIGNMENT_POLICY_TITLE: &str = "Chat stopped as a precaution"; @@ -41,6 +48,7 @@ impl ChatWidget { && !matches!( op, AppCommand::Interrupt + | AppCommand::RealtimeConversationStop { .. } | AppCommand::CleanBackgroundTerminals | AppCommand::OverrideTurnContext { .. } | AppCommand::ReloadUserConfig @@ -80,6 +88,10 @@ impl ChatWidget { { return; } + let retired_voice = self.realtime_conversation_is_running(); + if retired_voice { + self.stop_realtime_conversation(); + } let review = self.thread_id.zip(turn_id.clone()).zip(details).map( |((thread_id, turn_id), details)| { Arc::new(MisalignmentReview { @@ -89,7 +101,11 @@ impl ChatWidget { }) }, ); - self.misalignment_policy_violation = Some(MisalignmentViolation { turn_id, review }); + self.misalignment_policy_violation = Some(MisalignmentViolation { + turn_id, + review, + retired_voice, + }); self.input_queue.clear(); self.finalize_turn(); self.refresh_pending_input_preview(); @@ -211,11 +227,19 @@ impl ChatWidget { }); } - pub(crate) fn clear_misalignment_for_new_turn(&mut self, turn_id: &str) { + pub(crate) fn clear_misalignment_for_new_turn( + &mut self, + turn_id: &str, + source: MisalignmentTurnSource, + ) { if self .misalignment_policy_violation .as_ref() - .is_some_and(|violation| violation.turn_id.as_ref().is_some_and(|id| id != turn_id)) + .is_some_and(|violation| { + violation.turn_id.as_ref().is_some_and(|id| id != turn_id) + && (!violation.retired_voice + || source == MisalignmentTurnSource::AcknowledgedContinuation) + }) { self.misalignment_policy_violation = None; self.bottom_pane.dismiss_view_by_id(PRECAUTION_VIEW); diff --git a/codex-rs/tui/src/chatwidget/protocol.rs b/codex-rs/tui/src/chatwidget/protocol.rs index c2fc8039ef..d43adf4f53 100644 --- a/codex-rs/tui/src/chatwidget/protocol.rs +++ b/codex-rs/tui/src/chatwidget/protocol.rs @@ -61,7 +61,10 @@ impl ChatWidget { } ServerNotification::TurnStarted(notification) => { if replay_kind.is_none() { - self.clear_misalignment_for_new_turn(¬ification.turn.id); + self.clear_misalignment_for_new_turn( + ¬ification.turn.id, + MisalignmentTurnSource::ServerNotification, + ); } self.turn_lifecycle.last_turn_id = Some(notification.turn.id); self.last_non_retry_error = None; @@ -80,18 +83,43 @@ impl ChatWidget { self.handle_item_completed_notification(notification, replay_kind); } ServerNotification::AgentMessageDelta(notification) => { - self.on_agent_message_delta(notification.delta); + if !self.is_realtime_delegated_reasoning_turn(¬ification.turn_id) + && (from_replay + || !self.is_realtime_delegated_agent_item( + ¬ification.turn_id, + ¬ification.item_id, + )) + { + self.on_agent_message_delta(notification.delta); + } } ServerNotification::PlanDelta(notification) => self.on_plan_delta(notification.delta), ServerNotification::ReasoningSummaryTextDelta(notification) => { - self.on_agent_reasoning_delta(notification.delta); - } - ServerNotification::ReasoningTextDelta(notification) => { - if self.config.show_raw_agent_reasoning { + if !self.is_realtime_delegated_reasoning_item( + ¬ification.turn_id, + ¬ification.item_id, + ) { self.on_agent_reasoning_delta(notification.delta); } } - ServerNotification::ReasoningSummaryPartAdded(_) => self.on_reasoning_section_break(), + ServerNotification::ReasoningTextDelta(notification) => { + if self.config.show_raw_agent_reasoning + && !self.is_realtime_delegated_reasoning_item( + ¬ification.turn_id, + ¬ification.item_id, + ) + { + self.on_agent_reasoning_delta(notification.delta); + } + } + ServerNotification::ReasoningSummaryPartAdded(notification) => { + if !self.is_realtime_delegated_reasoning_item( + ¬ification.turn_id, + ¬ification.item_id, + ) { + self.on_reasoning_section_break(); + } + } ServerNotification::TerminalInteraction(notification) => { self.on_terminal_interaction(notification.process_id, notification.stdin) } @@ -235,6 +263,36 @@ impl ChatWidget { self.on_shutdown_complete(); } } + ServerNotification::ThreadRealtimeSdp(notification) => { + if !from_replay { + self.on_realtime_conversation_sdp(notification.sdp); + } + } + ServerNotification::ThreadRealtimeStarted(_) => { + if !from_replay { + self.on_realtime_conversation_started(); + } + } + ServerNotification::ThreadRealtimeTranscriptDelta(notification) => { + if !from_replay { + self.on_realtime_transcript_delta(notification.role, notification.delta); + } + } + ServerNotification::ThreadRealtimeTranscriptDone(notification) => { + if !from_replay { + self.on_realtime_transcript_done(notification.role, notification.text); + } + } + ServerNotification::ThreadRealtimeError(notification) => { + if !from_replay { + self.on_realtime_error(notification.message); + } + } + ServerNotification::ThreadRealtimeClosed(notification) => { + if !from_replay { + self.on_realtime_conversation_closed(notification.reason); + } + } ServerNotification::ServerRequestResolved(_) | ServerNotification::AccountUpdated(_) | ServerNotification::AccountRateLimitsUpdated(_) @@ -264,17 +322,11 @@ impl ChatWidget { | ServerNotification::TurnModerationMetadata(_) | ServerNotification::FuzzyFileSearchSessionUpdated(_) | ServerNotification::FuzzyFileSearchSessionCompleted(_) - | ServerNotification::ThreadRealtimeStarted(_) | ServerNotification::ThreadRealtimeItemAdded(_) | ServerNotification::ThreadRealtimeItemStarted(_) | ServerNotification::ThreadRealtimeItemTranscriptDelta(_) | ServerNotification::ThreadRealtimeItemCompleted(_) | ServerNotification::ThreadRealtimeOutputAudioDelta(_) - | ServerNotification::ThreadRealtimeError(_) - | ServerNotification::ThreadRealtimeClosed(_) - | ServerNotification::ThreadRealtimeSdp(_) - | ServerNotification::ThreadRealtimeTranscriptDelta(_) - | ServerNotification::ThreadRealtimeTranscriptDone(_) | ServerNotification::WindowsWorldWritableWarning(_) | ServerNotification::WindowsSandboxSetupCompleted(_) | ServerNotification::AccountLoginCompleted(_) @@ -310,7 +362,12 @@ impl ChatWidget { text, phase: Some(MessagePhase::FinalAnswer) | None, .. - } => Some((item.clone(), id.clone(), text.clone())), + } if !self + .is_realtime_delegated_reasoning_turn(¬ification.turn.id) + || !realtime::is_private_realtime_agent_item(item) => + { + Some((item.clone(), id.clone(), text.clone())) + } _ => None, }); if let Some((item, id, _)) = &last_agent_message @@ -329,6 +386,11 @@ impl ChatWidget { .map_or(ThreadItemRenderSource::Live, ThreadItemRenderSource::Replay), ); } + if replay_kind.is_none() + && let Some((item, _, _)) = &last_agent_message + { + self.speak_completed_realtime_delegation(¬ification.turn.id, item); + } self.last_non_retry_error = None; let completion = self.completion_cell(¬ification.turn, replay_kind); self.on_task_complete( @@ -355,7 +417,10 @@ impl ChatWidget { && error.codex_error_info == Some(AppServerCodexErrorInfo::MisalignmentPolicyViolation) { - self.on_misalignment_error(Some(notification.turn.id), error.misalignment); + self.on_misalignment_error( + Some(notification.turn.id.clone()), + error.misalignment, + ); self.last_non_retry_error = None; } else if self.last_non_retry_error.as_ref() == Some(&(notification.turn.id.clone(), error.message.clone())) @@ -373,6 +438,9 @@ impl ChatWidget { } TurnStatus::InProgress => {} } + if replay_kind.is_none() { + self.finish_realtime_turn(¬ification.turn.id); + } self.thread_usage.replaying_turn_completion = was_replaying_turn_completion; } @@ -382,6 +450,28 @@ impl ChatWidget { replay_kind: Option, ) { match notification.item { + ThreadItem::UserMessage { content, .. } if replay_kind.is_none() => { + self.note_realtime_user_item_started(¬ification.turn_id, &content); + } + ThreadItem::UserMessage { content, .. } + if realtime::realtime_delegation_input(&content).is_some() => + { + self.remember_realtime_delegated_reasoning_turn(¬ification.turn_id); + } + ThreadItem::AgentMessage { id, .. } if replay_kind.is_none() => { + self.is_realtime_delegated_agent_item(¬ification.turn_id, &id); + } + ThreadItem::Reasoning { id, .. } + if replay_kind.is_none() + && !self.is_realtime_delegated_reasoning_turn(¬ification.turn_id) => + { + // A later voice handoff can steer this turn without making an + // already-started typed reasoning item private. + self.realtime_conversation + .agent_items + .entry((notification.turn_id.clone(), id)) + .or_insert(realtime::RealtimeAgentItemOrigin::Typed); + } ThreadItem::ContextCompaction { id } if !matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)) => { @@ -441,6 +531,17 @@ impl ChatWidget { notification: ItemCompletedNotification, replay_kind: Option, ) { + if replay_kind.is_none() + && self.is_realtime_delegated_reasoning_turn(¬ification.turn_id) + && realtime::is_private_realtime_agent_item(¬ification.item) + && !matches!(¬ification.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. } + if matches!( + self.realtime_conversation.agent_items.get(&(notification.turn_id.clone(), id.clone())), + Some(realtime::RealtimeAgentItemOrigin::Typed) + )) + { + return; + } // Buffered live notifications can introduce questions; historical turn replay cannot. if replay_kind == Some(ReplayKind::ThreadSnapshot) && let ThreadItem::AgentMessage { diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs new file mode 100644 index 0000000000..c37a4e2565 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -0,0 +1,1785 @@ +//! TUI orchestration for an app-server-signaled, locally owned WebRTC voice session. +//! Completed captions and both speakers' partials stay bounded across widget replacement. + +use super::ChatWidget; +use super::HistoryCell; +use super::PARENT_OWNED_INPUT_MESSAGE; +use crate::app_command::AppCommand; +use crate::app_event::AppEvent; +use crate::history_cell; +use codex_app_server_protocol::ThreadItem; +use codex_app_server_protocol::UserInput; +use codex_features::Feature; +use codex_protocol::ThreadId; +use codex_protocol::models::MessagePhase; +use codex_realtime_webrtc::RealtimeWebrtcSession; +use codex_realtime_webrtc::RealtimeWebrtcSessionHandle; +use codex_realtime_webrtc::StartedRealtimeWebrtcSession; +use futures::future::AbortHandle; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hash; +use std::hash::Hasher; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::time::Duration; +use std::time::Instant; + +#[cfg(test)] +#[path = "realtime_tests.rs"] +pub(crate) mod tests; + +pub(crate) const MAX_TRANSCRIPT_BYTES: usize = 1024; +const MAX_PENDING_TRANSCRIPT_CELLS: usize = 32; +pub(crate) const MAX_REPLAY_TRANSCRIPT_CELLS: usize = MAX_PENDING_TRANSCRIPT_CELLS + 2; +const MAX_PENDING_SPEECH_DELIVERIES: usize = 16; +const MAX_RETAINED_DELEGATED_REASONING_TURNS: usize = 16; +const MAX_PENDING_SPEECH_ITEM_BYTES: usize = 64 * 1024; +const MAX_PENDING_SPEECH_TURN_ID_BYTES: usize = 512; +// Leave room for Core's optional backend prefix within its 1,000-token speech limit. +const MAX_SPEAKABLE_FINAL_TOKENS: usize = 990; +const AUDIO_METER_SEGMENTS: usize = 5; +const AUDIO_METER_NOISE_FLOOR: u16 = 512; +const AUDIO_METER_FULL_SCALE: u16 = 8192; +const MICROPHONE_METER_INTERVAL: Duration = Duration::from_millis(100); +const SPEAKER_ACTIVITY_HOLD: Duration = Duration::from_millis(500); +static NEXT_REALTIME_ATTEMPT_ID: AtomicU64 = AtomicU64::new(1); +static NEXT_REALTIME_SPEECH_DELIVERY_ID: AtomicU64 = AtomicU64::new(1); + +struct PendingRealtimeSpeech { + state: PendingSpeechState, + captioned: bool, + input_generation: u64, + thread_id: ThreadId, + turn_id: String, + item: ThreadItem, +} + +pub(crate) struct RealtimeTranscriptRecord { + pub(crate) role: String, + pub(crate) text: String, + pub(crate) complete: bool, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum PendingSpeechState { + AwaitingTurn, + Queued(u64), + Accepted, +} + +fn can_retain_realtime_speech(turn_id: &str, item: &ThreadItem) -> bool { + turn_id.len() <= MAX_PENDING_SPEECH_TURN_ID_BYTES + && serde_json::to_vec(item) + .is_ok_and(|encoded| encoded.len() <= MAX_PENDING_SPEECH_ITEM_BYTES) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum RealtimeConversationPhase { + #[default] + Inactive, + Starting, + Active, + Stopping, +} + +#[derive(Clone, Copy, Debug)] +pub(super) enum RealtimeAgentItemOrigin { + Typed, + Delegated { + may_speak: bool, + completed: bool, + suppressed_nonfinal: bool, + input_generation: u64, + }, +} + +#[derive(Clone, Copy, Debug)] +enum RealtimeTurnOrigin { + Typed { + input_generation: u64, + }, + Delegated { + may_speak: bool, + input_generation: u64, + }, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum StartupRetry { + #[default] + Available, + WaitingForStop, + Used, +} + +#[derive(Default)] +pub(super) struct RealtimeConversationUiState { + startup_retry: StartupRetry, + phase: RealtimeConversationPhase, + recover_late_transcripts: bool, + attempt_id: u64, + thread_id: Option, + pub(super) handle: Option, + startup_abort: Option, + backend_started: bool, + webrtc_connected: bool, + microphone_muted: bool, + microphone_level: usize, + speaker_level: usize, + speaker_active_until: Option, + speaker_suppression_generation: Option, + transcript_role: Option, + transcript: String, + // The other speaker's bounded partial while duplex deltas interleave. + interleaved_transcript: Option<(String, String)>, + transcript_input_generation: Option, + assistant_transcript_generation: Option, + assistant_caption_started_after_speech_queue: bool, + pub(super) live_transcript_cell: Option>, + pending_history_cells: VecDeque>, + accepted_transcripts: VecDeque, + latest_input_was_voice: bool, + input_generation: u64, + latest_voice_input_fingerprint: Option<(usize, u64)>, + pending_typed_input: Option, + turn_origins: HashMap, + delegated_reasoning_turns: VecDeque, + pub(super) agent_items: HashMap<(String, String), RealtimeAgentItemOrigin>, + pending_speech: VecDeque, +} + +pub(crate) fn realtime_delegation_input(items: &[UserInput]) -> Option<&str> { + let [ + UserInput::Text { + text, + text_elements, + }, + ] = items + else { + return None; + }; + if !text_elements.is_empty() { + return None; + } + let body = text + .trim() + .strip_prefix("") + .and_then(|body| body.strip_suffix(""))?; + let (_, input) = body.split_once("")?; + input.split_once("").map(|(input, _)| input) +} + +pub(crate) fn is_private_realtime_agent_item(item: &ThreadItem) -> bool { + if matches!(item, ThreadItem::Reasoning { .. }) { + return true; + } + let ThreadItem::AgentMessage { text, phase, .. } = item else { + return false; + }; + let text = text.trim_start(); + matches!(phase, Some(MessagePhase::Commentary)) + || (!matches!(phase, Some(MessagePhase::FinalAnswer)) + && (text.starts_with("[ANALYSIS]") || text.starts_with("[COMMENTARY]"))) +} + +pub(crate) fn realtime_delegation_display_text(input: &str) -> String { + input + .replace("<", "<") + .replace(">", ">") + .replace("&", "&") +} + +fn realtime_input_fingerprint(input: &str) -> (usize, u64) { + let input = input.trim(); + let mut hasher = DefaultHasher::new(); + input.hash(&mut hasher); + (input.len(), hasher.finish()) +} + +impl ChatWidget { + pub(crate) fn realtime_conversation_is_running(&self) -> bool { + self.realtime_conversation.phase != RealtimeConversationPhase::Inactive + } + + pub(crate) fn may_receive_realtime_transcripts(&self) -> bool { + self.realtime_conversation.phase != RealtimeConversationPhase::Inactive + || self.realtime_conversation.recover_late_transcripts + } + + pub(crate) fn toggle_realtime_conversation(&mut self) { + if self.realtime_conversation.phase == RealtimeConversationPhase::Stopping { + self.realtime_conversation.startup_retry = StartupRetry::Used; + self.add_info_message( + "Voice conversation is still stopping.".to_string(), + /*hint*/ None, + ); + return; + } + + if self.realtime_conversation.phase != RealtimeConversationPhase::Inactive { + self.stop_realtime_conversation(); + return; + } + + if self.has_misalignment_policy_violation() { + self.show_misalignment_policy_precaution(); + return; + } + + if self.side_conversation_active() { + self.add_error_message( + "Voice mode is unavailable in side conversations. Return to the main thread first." + .to_string(), + ); + return; + } + + if self.blocks_direct_input { + self.add_error_message(PARENT_OWNED_INPUT_MESSAGE.to_string()); + return; + } + + if !self.config.features.enabled(Feature::RealtimeConversation) { + self.add_error_message("Voice conversations are not enabled.".to_string()); + return; + } + + if !RealtimeWebrtcSession::is_supported() { + self.add_error_message( + "Voice requires macOS, an MSVC-based Windows build, or a glibc-based Linux build." + .to_string(), + ); + return; + } + + let Some(thread_id) = self.thread_id() else { + self.add_error_message("Start a conversation before using voice mode.".to_string()); + return; + }; + + self.start_realtime_conversation(thread_id); + } + + fn start_realtime_conversation(&mut self, thread_id: ThreadId) { + self.realtime_conversation.recover_late_transcripts = false; + self.realtime_conversation.attempt_id = + NEXT_REALTIME_ATTEMPT_ID.fetch_add(1, Ordering::Relaxed); + self.realtime_conversation.thread_id = Some(thread_id); + let attempt_id = self.realtime_conversation.attempt_id; + let (startup_abort, abort_registration) = AbortHandle::new_pair(); + self.realtime_conversation.startup_abort = Some(startup_abort); + self.realtime_conversation.phase = RealtimeConversationPhase::Starting; + self.update_realtime_footer(); + let app_event_tx = self.app_event_tx.clone(); + std::thread::spawn(move || { + let result = + RealtimeWebrtcSession::start(abort_registration).map_err(|error| error.to_string()); + app_event_tx.send(AppEvent::RealtimeWebrtcOfferCreated { + thread_id, + attempt_id, + result, + }); + }); + self.request_redraw(); + } + + pub(super) fn stop_realtime_conversation(&mut self) { + self.realtime_conversation.startup_retry = StartupRetry::Used; + if matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Inactive | RealtimeConversationPhase::Stopping + ) { + return; + } + self.restore_all_undelivered_realtime_speech(); + + // Stopping may prevent final transcript events from arriving. + self.finish_realtime_partial_transcripts(); + + if let Some(handle) = self.realtime_conversation.handle.take() { + handle.close(); + self.realtime_conversation.phase = RealtimeConversationPhase::Stopping; + self.set_footer_hint_override(/*items*/ None); + let Some(thread_id) = self.realtime_conversation.thread_id else { + self.reset_realtime_conversation(); + return; + }; + if !self.submit_op(AppCommand::RealtimeConversationStop { thread_id }) { + self.reset_realtime_conversation(); + } + } else { + if let Some(thread_id) = self.reset_realtime_conversation() { + self.submit_op(AppCommand::RealtimeConversationStop { thread_id }); + } + } + self.request_redraw(); + } + + pub(super) fn toggle_realtime_microphone(&mut self) { + let muted = !self.realtime_conversation.microphone_muted; + if let Some(handle) = self.realtime_conversation.handle.as_ref() { + if let Err(error) = handle.set_microphone_muted(muted) { + self.on_realtime_error(format!("Failed to update microphone: {error}")); + return; + } + } else if self.realtime_conversation.phase != RealtimeConversationPhase::Starting { + self.add_error_message("Start voice mode before muting the microphone.".to_string()); + return; + } + + self.realtime_conversation.microphone_muted = muted; + self.realtime_conversation.microphone_level = 0; + self.update_realtime_footer(); + } + + pub(crate) fn on_realtime_webrtc_offer_created( + &mut self, + thread_id: ThreadId, + attempt_id: u64, + result: Result, + ) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Starting + || self.realtime_conversation.attempt_id != attempt_id + { + if let Ok(offer) = result { + offer.handle.close(); + } + return; + } + + let offer = match result { + Ok(offer) => offer, + Err(error) => { + self.on_realtime_error(format!("Failed to start voice mode: {error}")); + return; + } + }; + self.realtime_conversation.startup_abort = None; + if self.realtime_conversation.microphone_muted + && let Err(error) = offer.handle.set_microphone_muted(/*muted*/ true) + { + offer.handle.close(); + self.on_realtime_error(format!("Failed to restore microphone mute: {error}")); + return; + } + self.realtime_conversation.handle = Some(offer.handle); + self.frame_requester + .schedule_frame_in(MICROPHONE_METER_INTERVAL); + if !self.submit_op(AppCommand::RealtimeConversationStart { + thread_id, + offer_sdp: offer.offer_sdp.into(), + }) { + self.reset_realtime_conversation(); + } + } + + pub(super) fn on_realtime_conversation_sdp(&mut self, answer_sdp: String) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Starting { + return; + } + let Some(handle) = self.realtime_conversation.handle.clone() else { + return; + }; + let Some(thread_id) = self.thread_id() else { + return; + }; + let attempt_id = self.realtime_conversation.attempt_id; + let app_event_tx = self.app_event_tx.clone(); + std::thread::spawn(move || { + let result = handle.apply_answer_sdp(answer_sdp); + app_event_tx.send(AppEvent::RealtimeWebrtcConnected { + thread_id, + attempt_id, + result, + }); + }); + } + + pub(super) fn on_realtime_conversation_started(&mut self) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Starting { + return; + } + self.realtime_conversation.backend_started = true; + self.maybe_activate_realtime_conversation(); + } + + pub(crate) fn on_realtime_webrtc_connected( + &mut self, + attempt_id: u64, + result: Result<(), codex_realtime_webrtc::ConnectionError>, + ) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Starting + || self.realtime_conversation.attempt_id != attempt_id + { + return; + } + if let Err(error) = result { + if error == codex_realtime_webrtc::ConnectionError::NegotiationTimedOut + && self.realtime_conversation.startup_retry == StartupRetry::Available + && let Some(thread_id) = self.realtime_conversation.thread_id + { + if let Some(handle) = self.realtime_conversation.handle.take() { + handle.close(); + } + self.realtime_conversation.phase = RealtimeConversationPhase::Stopping; + self.realtime_conversation.startup_retry = StartupRetry::WaitingForStop; + if !self.submit_op(AppCommand::RealtimeConversationStop { thread_id }) { + self.reset_realtime_conversation(); + } else { + self.add_info_message( + "Voice connection timed out. Retrying once after cleanup.".into(), + /*hint*/ None, + ); + } + return; + } + self.on_realtime_error(format!("Failed to connect voice mode: {error}")); + return; + } + + self.realtime_conversation.webrtc_connected = true; + self.maybe_activate_realtime_conversation(); + } + + fn maybe_activate_realtime_conversation(&mut self) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Starting + || !self.realtime_conversation.backend_started + || !self.realtime_conversation.webrtc_connected + { + return; + } + self.realtime_conversation.phase = RealtimeConversationPhase::Active; + let running_delegation = + self.turn_lifecycle + .last_turn_id + .as_deref() + .is_some_and(|turn_id| { + matches!( + self.realtime_conversation.turn_origins.get(turn_id), + Some(RealtimeTurnOrigin::Delegated { + may_speak: true, + .. + }) + ) + }); + self.realtime_conversation.latest_input_was_voice = + !self.turn_lifecycle.agent_turn_running || running_delegation; + if self.turn_lifecycle.agent_turn_running + && let Some(turn_id) = self.turn_lifecycle.last_turn_id.clone() + { + self.realtime_conversation + .turn_origins + .entry(turn_id) + .or_insert(RealtimeTurnOrigin::Typed { + input_generation: self.realtime_conversation.input_generation, + }); + } + self.update_realtime_footer(); + self.frame_requester + .schedule_frame_in(MICROPHONE_METER_INTERVAL); + self.add_info_message( + "Voice conversation started.".to_string(), + Some("Use /voice mute to mute or /voice to stop.".to_string()), + ); + } + + pub(crate) fn is_current_realtime_attempt( + &self, + thread_id: ThreadId, + attempt_id: u64, + input_generation: u64, + ) -> bool { + self.realtime_conversation.phase == RealtimeConversationPhase::Active + && self.realtime_conversation.thread_id == Some(thread_id) + && self.realtime_conversation.attempt_id == attempt_id + && self.realtime_conversation.input_generation == input_generation + && self.realtime_conversation.latest_input_was_voice + } + + pub(super) fn note_realtime_typed_input(&mut self, text: &str) { + self.invalidate_realtime_voice_input(); + if self.realtime_conversation.phase == RealtimeConversationPhase::Active { + self.realtime_conversation.pending_typed_input = Some(text.to_string()); + } + } + + fn invalidate_realtime_voice_input(&mut self) { + if self.realtime_conversation.phase != RealtimeConversationPhase::Active { + return; + } + self.realtime_conversation.latest_input_was_voice = false; + self.realtime_conversation.input_generation = self + .realtime_conversation + .input_generation + .wrapping_add(/*rhs*/ 1); + // Typing supersedes the old voice answer. Suppress the helper before + // another queued audio frame can play, even when the mic is muted. + self.realtime_conversation.speaker_suppression_generation = + Some(self.realtime_conversation.input_generation); + if let Some(handle) = self.realtime_conversation.handle.as_ref() { + handle.set_speaker_suppressed(/*suppressed*/ true); + } + self.realtime_conversation.speaker_level = 0; + self.realtime_conversation.speaker_active_until = None; + self.update_realtime_footer(); + for origin in self.realtime_conversation.turn_origins.values_mut() { + if let RealtimeTurnOrigin::Delegated { may_speak, .. } = origin { + *may_speak = false; + } + } + for origin in self.realtime_conversation.agent_items.values_mut() { + if let RealtimeAgentItemOrigin::Delegated { may_speak, .. } = origin { + *may_speak = false; + } + } + } + + fn release_realtime_speaker(&mut self) { + self.realtime_conversation.speaker_suppression_generation = None; + if let Some(handle) = self.realtime_conversation.handle.as_ref() { + handle.set_speaker_suppressed(/*suppressed*/ false); + } + } + + fn suppress_realtime_speaker(&mut self) { + self.realtime_conversation.speaker_suppression_generation = + Some(self.realtime_conversation.input_generation); + if let Some(handle) = self.realtime_conversation.handle.as_ref() { + handle.set_speaker_suppressed(/*suppressed*/ true); + } + self.realtime_conversation.speaker_level = 0; + self.realtime_conversation.speaker_active_until = None; + self.update_realtime_footer(); + } + + fn resume_realtime_speaker_for(&mut self, role: &str, text: &str) { + if role == "assistant" + && !text.trim().is_empty() + && self.realtime_conversation.assistant_transcript_generation + == Some(self.realtime_conversation.input_generation) + && self.realtime_conversation.latest_input_was_voice + && self.realtime_conversation.speaker_suppression_generation + == Some(self.realtime_conversation.input_generation) + { + self.release_realtime_speaker(); + } + } + + pub(super) fn note_realtime_user_item_started(&mut self, turn_id: &str, items: &[UserInput]) { + if !matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ) { + if realtime_delegation_input(items).is_some() { + self.remember_realtime_delegated_reasoning_turn(turn_id); + } + return; + } + let matches_local_typed_input = self + .realtime_conversation + .pending_typed_input + .as_ref() + .is_some_and(|pending| { + items + .iter() + .any(|item| matches!(item, UserInput::Text { text, .. } if text == pending)) + }); + if matches_local_typed_input { + self.realtime_conversation.pending_typed_input = None; + } + if let Some(input) = realtime_delegation_input(items) + && !matches_local_typed_input + { + let transcript_tail_flush = items.iter().any(|item| { + matches!(item, UserInput::Text { text, .. } + if text.contains("transcript_tail_flush")) + }); + let stale_voice_input = !self.realtime_conversation.latest_input_was_voice + && self + .realtime_conversation + .latest_voice_input_fingerprint + .is_some_and(|latest| { + latest + == realtime_input_fingerprint(&realtime_delegation_display_text(input)) + }); + let (typed_turn_generation, delegated_turn_generation) = + match self.realtime_conversation.turn_origins.get(turn_id) { + Some(RealtimeTurnOrigin::Typed { input_generation }) => { + (Some(*input_generation), None) + } + Some(RealtimeTurnOrigin::Delegated { + input_generation, .. + }) => (None, Some(*input_generation)), + None => (None, None), + }; + let voice_supersedes_typed_turn = typed_turn_generation.is_some_and(|generation| { + generation != self.realtime_conversation.input_generation + || !self.realtime_conversation.latest_input_was_voice + }) && !transcript_tail_flush + && !stale_voice_input; + let typed_turn = typed_turn_generation.is_some() && !voice_supersedes_typed_turn; + let may_speak = !transcript_tail_flush && !stale_voice_input && !typed_turn; + if may_speak { + let replaces_current_delegation = self + .realtime_conversation + .turn_origins + .values() + .any(|origin| { + matches!( + origin, + RealtimeTurnOrigin::Delegated { + may_speak: true, + input_generation, + } if *input_generation == self.realtime_conversation.input_generation + ) + }); + if !self.realtime_conversation.latest_input_was_voice || replaces_current_delegation + { + self.realtime_conversation.input_generation = self + .realtime_conversation + .input_generation + .wrapping_add(/*rhs*/ 1); + if self + .realtime_conversation + .speaker_suppression_generation + .is_some() + { + self.realtime_conversation.speaker_suppression_generation = + Some(self.realtime_conversation.input_generation); + } + } + self.realtime_conversation.latest_input_was_voice = true; + } + let delegated_origin = RealtimeTurnOrigin::Delegated { + may_speak, + input_generation: self.realtime_conversation.input_generation, + }; + let supersedes_previous_delegation = may_speak + && delegated_turn_generation.is_some_and(|generation| { + generation != self.realtime_conversation.input_generation + }); + if voice_supersedes_typed_turn || supersedes_previous_delegation { + self.realtime_conversation + .turn_origins + .insert(turn_id.to_string(), delegated_origin); + } else { + self.realtime_conversation + .turn_origins + .entry(turn_id.to_string()) + .or_insert(delegated_origin); + } + if matches!( + self.realtime_conversation.turn_origins.get(turn_id), + Some(RealtimeTurnOrigin::Delegated { .. }) + ) { + self.remember_realtime_delegated_reasoning_turn(turn_id); + } + return; + } + + self.invalidate_realtime_voice_input(); + self.realtime_conversation.turn_origins.insert( + turn_id.to_string(), + RealtimeTurnOrigin::Typed { + input_generation: self.realtime_conversation.input_generation, + }, + ); + self.realtime_conversation + .delegated_reasoning_turns + .retain(|saved| saved != turn_id); + } + + pub(crate) fn remember_realtime_delegated_reasoning_turn(&mut self, turn_id: &str) { + if turn_id.len() > 512 + || self + .realtime_conversation + .delegated_reasoning_turns + .iter() + .any(|saved| saved == turn_id) + { + return; + } + self.realtime_conversation + .delegated_reasoning_turns + .push_back(turn_id.to_string()); + if self.realtime_conversation.delegated_reasoning_turns.len() + > MAX_RETAINED_DELEGATED_REASONING_TURNS + { + self.realtime_conversation + .delegated_reasoning_turns + .pop_front(); + } + } + + pub(super) fn is_realtime_delegated_reasoning_turn(&self, turn_id: &str) -> bool { + self.realtime_conversation + .delegated_reasoning_turns + .iter() + .any(|saved| saved == turn_id) + } + + pub(super) fn is_realtime_delegated_reasoning_item( + &self, + turn_id: &str, + item_id: &str, + ) -> bool { + self.is_realtime_delegated_reasoning_turn(turn_id) + && !matches!( + self.realtime_conversation + .agent_items + .get(&(turn_id.to_string(), item_id.to_string())), + Some(RealtimeAgentItemOrigin::Typed) + ) + } + + pub(super) fn should_hide_realtime_delegation(&self, turn_id: &str) -> bool { + matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ) && matches!( + self.realtime_conversation.turn_origins.get(turn_id), + Some(RealtimeTurnOrigin::Delegated { .. }) + ) + } + + pub(super) fn is_realtime_delegated_agent_item( + &mut self, + turn_id: &str, + item_id: &str, + ) -> bool { + if !matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ) { + return false; + } + let default_origin = match self.realtime_conversation.turn_origins.get(turn_id) { + Some(RealtimeTurnOrigin::Delegated { + may_speak, + input_generation, + }) => RealtimeAgentItemOrigin::Delegated { + may_speak: *may_speak, + completed: false, + suppressed_nonfinal: false, + input_generation: *input_generation, + }, + Some(RealtimeTurnOrigin::Typed { .. }) | None => RealtimeAgentItemOrigin::Typed, + }; + matches!( + self.realtime_conversation + .agent_items + .entry((turn_id.to_string(), item_id.to_string())) + .or_insert(default_origin), + RealtimeAgentItemOrigin::Delegated { .. } + ) + } + + pub(super) fn complete_realtime_delegated_agent_item( + &mut self, + turn_id: &str, + item: &ThreadItem, + from_replay: bool, + ) -> bool { + let ThreadItem::AgentMessage { + id: item_id, + text, + phase, + questions, + .. + } = item + else { + return false; + }; + // The ordinary completion path opens the structured question editor. + // A question-bearing answer must remain visible instead of being speech-only. + if questions.is_some() { + return false; + } + if from_replay || !self.is_realtime_delegated_agent_item(turn_id, item_id) { + return false; + } + let Some(RealtimeAgentItemOrigin::Delegated { + may_speak, + completed, + suppressed_nonfinal, + .. + }) = self + .realtime_conversation + .agent_items + .get_mut(&(turn_id.to_string(), item_id.to_string())) + else { + return false; + }; + if *completed { + return true; + } + // The delegated agent's commentary and reasoning are private to the + // voice handoff. Never put them in the fallback queue, even when the + // item is too large to retain. + let trimmed = text.trim(); + let explicitly_final = matches!(phase, Some(MessagePhase::FinalAnswer)); + if is_private_realtime_agent_item(item) + || (!explicitly_final && trimmed == "[FINAL]") + || trimmed.is_empty() + { + *completed = true; + *suppressed_nonfinal = true; + return true; + } + // Typing ends the old item's speech ownership, but it does not make + // private voice commentary into typed output. Only a visible answer + // returns to the ordinary completion path. + if !*may_speak { + return false; + } + // A final item that cannot be retained for delivery recovery stays in + // ordinary history instead of becoming a voice-only answer. + if !can_retain_realtime_speech(turn_id, item) { + return false; + } + let Some(thread_id) = self.realtime_conversation.thread_id else { + return false; + }; + let input_generation = self.realtime_conversation.input_generation; + *completed = true; + self.transcript.last_completed_agent_message = + Some((turn_id.to_string(), item_id.to_string())); + if self.realtime_conversation.pending_speech.len() >= MAX_PENDING_SPEECH_DELIVERIES + && let Some(oldest) = self.realtime_conversation.pending_speech.pop_front() + { + self.restore_realtime_speech(oldest); + } + self.realtime_conversation + .pending_speech + .push_back(PendingRealtimeSpeech { + state: PendingSpeechState::AwaitingTurn, + captioned: false, + input_generation, + thread_id, + turn_id: turn_id.to_string(), + item: item.clone(), + }); + + true + } + + pub(super) fn speak_completed_realtime_delegation(&mut self, turn_id: &str, item: &ThreadItem) { + let ThreadItem::AgentMessage { + id: item_id, text, .. + } = item + else { + return; + }; + let Some(RealtimeAgentItemOrigin::Delegated { + may_speak: true, + completed: true, + suppressed_nonfinal, + input_generation, + }) = self + .realtime_conversation + .agent_items + .get(&(turn_id.to_string(), item_id.to_string())) + else { + return; + }; + let input_generation = *input_generation; + let suppressed_nonfinal = *suppressed_nonfinal; + if input_generation != self.realtime_conversation.input_generation { + return; + } + let trimmed = text.trim(); + if is_private_realtime_agent_item(item) + || trimmed.is_empty() + || !self.realtime_conversation.latest_input_was_voice + { + return; + } + let Some(thread_id) = self.realtime_conversation.thread_id else { + return; + }; + if self.thread_id() != Some(thread_id) { + return; + } + let text = trimmed + .strip_prefix("[FINAL]") + .map(str::trim_start) + .unwrap_or(trimmed) + .to_string(); + if text.is_empty() { + return; + } + let Some(RealtimeTurnOrigin::Delegated { + may_speak, + input_generation: turn_input_generation, + }) = self.realtime_conversation.turn_origins.get_mut(turn_id) + else { + return; + }; + if !*may_speak || *turn_input_generation != input_generation { + return; + } + *may_speak = false; + if !can_retain_realtime_speech(turn_id, item) + || codex_utils_string::approx_token_count(&text) > MAX_SPEAKABLE_FINAL_TOKENS + { + self.remove_waiting_realtime_speech(turn_id, item_id); + self.finish_realtime_turn(turn_id); + self.handle_thread_item( + item.clone(), + turn_id.to_string(), + super::ThreadItemRenderSource::Live, + ); + return; + } + let delivery_id = NEXT_REALTIME_SPEECH_DELIVERY_ID.fetch_add(1, Ordering::Relaxed); + let index = self + .realtime_conversation + .pending_speech + .iter_mut() + .position(|pending| { + pending.turn_id == turn_id + && matches!(&pending.item, ThreadItem::AgentMessage { id, .. } if id == item_id) + && pending.state == PendingSpeechState::AwaitingTurn + }); + let index = match index { + Some(index) => index, + None if suppressed_nonfinal => { + if self.realtime_conversation.pending_speech.len() >= MAX_PENDING_SPEECH_DELIVERIES + && let Some(oldest) = self.realtime_conversation.pending_speech.pop_front() + { + self.restore_realtime_speech(oldest); + } + self.realtime_conversation + .pending_speech + .push_back(PendingRealtimeSpeech { + state: PendingSpeechState::AwaitingTurn, + captioned: false, + input_generation, + thread_id, + turn_id: turn_id.to_string(), + item: item.clone(), + }); + self.realtime_conversation.pending_speech.len() - 1 + } + None => return, + }; + let pending = &mut self.realtime_conversation.pending_speech[index]; + pending.item = item.clone(); + pending.state = PendingSpeechState::Queued(delivery_id); + if !self.submit_op(AppCommand::RealtimeConversationSpeech { + thread_id, + attempt_id: self.realtime_conversation.attempt_id, + input_generation, + delivery_id, + text: text.into(), + }) { + self.restore_undelivered_realtime_speech(delivery_id); + self.on_realtime_error("Failed to deliver the voice response.".to_string()); + } else { + self.realtime_conversation.pending_speech.retain(|pending| { + pending.turn_id != turn_id || pending.state != PendingSpeechState::AwaitingTurn + }); + } + } + + pub(crate) fn has_pending_realtime_speech(&self, delivery_id: u64) -> bool { + self.realtime_conversation + .pending_speech + .iter() + .any(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id)) + } + + pub(crate) fn take_undelivered_realtime_speech_for_replay( + &mut self, + ) -> Vec<(ThreadId, String, ThreadItem)> { + self.realtime_conversation + .pending_speech + .drain(..) + .filter(|delivery| !delivery.captioned) + .map(|delivery| (delivery.thread_id, delivery.turn_id, delivery.item)) + .collect() + } + + pub(crate) fn take_realtime_transcript_cells_for_replay( + &mut self, + ) -> VecDeque { + let mut records = std::mem::take(&mut self.realtime_conversation.accepted_transcripts); + // Accepted records include cells already flushed into the old widget as + // well as those still deferred behind another assistant stream. + self.realtime_conversation.pending_history_cells.clear(); + let mut retain_partial = |role: String, text: String| { + if text.trim().is_empty() { + return; + } + if let Some(partial) = records + .iter_mut() + .rev() + .find(|record| record.role == role && !record.complete) + { + partial.text = text; + return; + } + if records.len() >= MAX_REPLAY_TRANSCRIPT_CELLS { + records.pop_front(); + } + records.push_back(RealtimeTranscriptRecord { + role, + text, + complete: false, + }); + }; + if let Some((role, text)) = self.realtime_conversation.interleaved_transcript.take() { + retain_partial(role, text); + } + if let Some(role) = self.realtime_conversation.transcript_role.take() { + let text = std::mem::take(&mut self.realtime_conversation.transcript); + retain_partial(role, text); + self.realtime_conversation.live_transcript_cell = None; + } + records + } + + pub(crate) fn restore_realtime_transcript_cells( + &mut self, + records: VecDeque, + ) { + self.realtime_conversation.recover_late_transcripts = true; + for record in records { + if self.realtime_conversation.accepted_transcripts.len() >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.accepted_transcripts.pop_front(); + } + if record.complete { + if self.realtime_conversation.pending_history_cells.len() + >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.pending_history_cells.pop_front(); + } + let cell = self.realtime_transcript_history_cell(&record.role, &record.text); + self.realtime_conversation + .pending_history_cells + .push_back(cell); + } else { + // Keep an unfinished caption editable until its late completion or close. + self.on_realtime_transcript_delta(record.role.clone(), record.text.clone()); + } + self.realtime_conversation + .accepted_transcripts + .push_back(record); + } + self.flush_realtime_transcript_history(); + } + + pub(crate) fn render_undelivered_realtime_speech( + &mut self, + thread_id: ThreadId, + turn_id: String, + item: ThreadItem, + ) { + if self.thread_id() == Some(thread_id) { + self.handle_thread_item(item, turn_id, super::ThreadItemRenderSource::Live); + } + } + + pub(crate) fn accept_realtime_speech(&mut self, delivery_id: u64) { + if let Some(delivery) = self + .realtime_conversation + .pending_speech + .iter_mut() + .find(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id)) + { + delivery.state = PendingSpeechState::Accepted; + } + self.realtime_conversation + .pending_speech + .retain(|delivery| !delivery.captioned); + } + + pub(crate) fn restore_undelivered_realtime_speech(&mut self, delivery_id: u64) { + if let Some(index) = self + .realtime_conversation + .pending_speech + .iter() + .position(|delivery| delivery.state == PendingSpeechState::Queued(delivery_id)) + && let Some(delivery) = self.realtime_conversation.pending_speech.remove(index) + { + self.restore_realtime_speech(delivery); + } + } + + fn restore_realtime_speech(&mut self, delivery: PendingRealtimeSpeech) { + if delivery.captioned { + return; + } + if self.thread_id() != Some(delivery.thread_id) { + return; + } + self.forget_realtime_turn_origin(&delivery.turn_id); + self.handle_thread_item( + delivery.item, + delivery.turn_id, + super::ThreadItemRenderSource::Live, + ); + } + + fn restore_all_undelivered_realtime_speech(&mut self) { + while let Some(delivery) = self.realtime_conversation.pending_speech.pop_front() { + self.restore_realtime_speech(delivery); + } + } + + pub(super) fn finish_realtime_turn(&mut self, turn_id: &str) { + let mut waiting = Vec::new(); + let mut index = 0; + while index < self.realtime_conversation.pending_speech.len() { + if self.realtime_conversation.pending_speech[index].turn_id == turn_id + && self.realtime_conversation.pending_speech[index].state + == PendingSpeechState::AwaitingTurn + { + if let Some(delivery) = self.realtime_conversation.pending_speech.remove(index) { + waiting.push(delivery); + } + } else { + index += 1; + } + } + self.forget_realtime_turn_origin(turn_id); + for delivery in waiting { + if self.thread_id() == Some(delivery.thread_id) { + self.handle_thread_item( + delivery.item, + delivery.turn_id, + super::ThreadItemRenderSource::Live, + ); + } + } + } + + fn remove_waiting_realtime_speech(&mut self, turn_id: &str, item_id: &str) { + self.realtime_conversation.pending_speech.retain(|pending| { + pending.turn_id != turn_id + || pending.state != PendingSpeechState::AwaitingTurn + || !matches!(&pending.item, ThreadItem::AgentMessage { id, .. } if id == item_id) + }); + } + + fn forget_realtime_turn_origin(&mut self, turn_id: &str) { + self.realtime_conversation.turn_origins.remove(turn_id); + self.realtime_conversation + .delegated_reasoning_turns + .retain(|saved| saved != turn_id); + self.realtime_conversation + .agent_items + .retain(|(item_turn_id, _), _| item_turn_id != turn_id); + } + + pub(super) fn on_realtime_transcript_delta(&mut self, role: String, delta: String) { + if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive + && !self.realtime_conversation.recover_late_transcripts + { + return; + } + let active = matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ); + // Duplex transcript chunks can interleave. Display role changes are not input boundaries. + if active + && role == "user" + && self + .realtime_conversation + .transcript_input_generation + .is_none() + { + self.realtime_conversation.input_generation = self + .realtime_conversation + .input_generation + .wrapping_add(/*rhs*/ 1); + self.realtime_conversation.transcript_input_generation = + Some(self.realtime_conversation.input_generation); + self.suppress_realtime_speaker(); + } + if active + && role == "assistant" + && !delta.trim().is_empty() + && self + .realtime_conversation + .assistant_transcript_generation + .is_none() + { + self.realtime_conversation.assistant_transcript_generation = + Some(self.realtime_conversation.input_generation); + self.realtime_conversation + .assistant_caption_started_after_speech_queue = self + .realtime_conversation + .pending_speech + .iter() + .any(|delivery| { + delivery.input_generation == self.realtime_conversation.input_generation + && delivery.state != PendingSpeechState::AwaitingTurn + }); + } + if active { + self.resume_realtime_speaker_for(&role, &delta); + } + if active + && role == "user" + && self.realtime_conversation.transcript_input_generation + == Some(self.realtime_conversation.input_generation) + { + self.realtime_conversation.latest_input_was_voice = true; + } + if self.realtime_conversation.transcript_role.as_deref() != Some(role.as_str()) { + let previous = self.realtime_conversation.transcript_role.take(); + let previous_text = std::mem::take(&mut self.realtime_conversation.transcript); + let saved = self.realtime_conversation.interleaved_transcript.take(); + self.realtime_conversation.transcript = match saved { + Some((saved_role, saved_text)) if saved_role == role => saved_text, + _ => String::new(), + }; + if let Some(previous_role) = previous { + self.realtime_conversation.interleaved_transcript = + Some((previous_role, previous_text)); + } + self.realtime_conversation.transcript_role = Some(role); + } + self.realtime_conversation.transcript.push_str(&delta); + if self.realtime_conversation.transcript.len() > MAX_TRANSCRIPT_BYTES { + let mut start = self.realtime_conversation.transcript.len() - MAX_TRANSCRIPT_BYTES; + while !self + .realtime_conversation + .transcript + .is_char_boundary(start) + { + start += 1; + } + self.realtime_conversation.transcript.drain(..start); + } + let role = self + .realtime_conversation + .transcript_role + .as_deref() + .unwrap_or(""); + self.realtime_conversation.live_transcript_cell = Some( + self.realtime_transcript_history_cell(role, &self.realtime_conversation.transcript), + ); + self.bump_active_cell_revision(); + self.request_redraw(); + } + + pub(super) fn on_realtime_transcript_done(&mut self, role: String, mut text: String) { + if matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Inactive | RealtimeConversationPhase::Stopping + ) { + if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive + && !self.realtime_conversation.recover_late_transcripts + { + return; + } + if self.realtime_conversation.transcript_role.as_deref() == Some(role.as_str()) { + self.realtime_conversation.transcript_role = None; + self.realtime_conversation.transcript.clear(); + self.realtime_conversation.live_transcript_cell = None; + self.bump_active_cell_revision(); + } else if self + .realtime_conversation + .interleaved_transcript + .as_ref() + .is_some_and(|(saved_role, _)| saved_role == &role) + { + self.realtime_conversation.interleaved_transcript = None; + } + if text.trim().is_empty() { + if let Some(index) = self + .realtime_conversation + .accepted_transcripts + .iter() + .rposition(|record| record.role == role && !record.complete) + { + self.realtime_conversation + .accepted_transcripts + .remove(index); + } + return; + } + if text.len() > MAX_TRANSCRIPT_BYTES { + let mut end = MAX_TRANSCRIPT_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + let display_text = if let Some(partial) = self + .realtime_conversation + .accepted_transcripts + .iter_mut() + .rev() + .find(|record| record.role == role && !record.complete) + { + partial.text = text.clone(); + partial.complete = true; + text.clone() + } else { + // A separate final can share a prefix with an earlier caption. + // Show it in full rather than guessing it extends that caption. + if self.realtime_conversation.accepted_transcripts.len() + >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.accepted_transcripts.pop_front(); + } + self.realtime_conversation.accepted_transcripts.push_back( + RealtimeTranscriptRecord { + role: role.clone(), + text: text.clone(), + complete: true, + }, + ); + text.clone() + }; + if display_text.is_empty() { + return; + } + if self.realtime_conversation.pending_history_cells.len() >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.pending_history_cells.pop_front(); + } + let cell = self.realtime_transcript_history_cell(&role, &display_text); + self.realtime_conversation + .pending_history_cells + .push_back(cell); + self.flush_realtime_transcript_history(); + return; + } + if !matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ) { + return; + } + let has_text = !text.trim().is_empty(); + if role == "assistant" && has_text { + // appendSpeech's RPC acknowledgement only means the request was + // queued. A subsequent assistant caption is the closest signal + // available here that its answer reached the voice session. The + // protocol does not identify which speech request produced it. + // A done-only caption has no generation owner, so it cannot retire + // a newer answer's fallback. + if self + .realtime_conversation + .assistant_caption_started_after_speech_queue + && let Some(caption_generation) = + self.realtime_conversation.assistant_transcript_generation + && let Some(delivery) = + self.realtime_conversation + .pending_speech + .iter_mut() + .find(|delivery| { + !delivery.captioned + && delivery.input_generation == caption_generation + && delivery.state != PendingSpeechState::AwaitingTurn + && matches!(&delivery.item, ThreadItem::AgentMessage { text: answer, .. } + if answer.trim().strip_prefix("[FINAL]").unwrap_or(answer.trim()) + .split_whitespace().eq(text.split_whitespace())) + }) + { + delivery.captioned = true; + } + self.realtime_conversation + .pending_speech + .retain(|delivery| { + !delivery.captioned || matches!(delivery.state, PendingSpeechState::Queued(_)) + }); + } + if has_text + && self + .realtime_conversation + .transcript_input_generation + .is_none() + { + self.resume_realtime_speaker_for(&role, &text); + } + // Check ownership before retiring the old output: its final chunk must not unmute it. + if role == "assistant" { + self.realtime_conversation.assistant_transcript_generation = None; + self.realtime_conversation + .assistant_caption_started_after_speech_queue = false; + } + let voice_input_fingerprint = (role == "user").then(|| realtime_input_fingerprint(&text)); + if role == "user" { + match self + .realtime_conversation + .transcript_input_generation + .take() + { + Some(generation) => { + if generation == self.realtime_conversation.input_generation { + self.realtime_conversation.latest_input_was_voice = has_text; + if !has_text { + self.release_realtime_speaker(); + } + } + } + None => { + let old_superseded_transcript = + !self.realtime_conversation.latest_input_was_voice + && self + .realtime_conversation + .latest_voice_input_fingerprint + .is_some_and(|latest| Some(latest) == voice_input_fingerprint); + if old_superseded_transcript || !has_text { + return; + } + self.realtime_conversation.input_generation = self + .realtime_conversation + .input_generation + .wrapping_add(/*rhs*/ 1); + self.realtime_conversation.latest_input_was_voice = true; + self.suppress_realtime_speaker(); + } + } + } + if self.realtime_conversation.transcript_role.as_deref() == Some(role.as_str()) { + self.realtime_conversation.transcript.clear(); + self.realtime_conversation.transcript_role = None; + self.realtime_conversation.live_transcript_cell = None; + self.bump_active_cell_revision(); + } else if self + .realtime_conversation + .interleaved_transcript + .as_ref() + .is_some_and(|(saved_role, _)| saved_role == &role) + { + self.realtime_conversation.interleaved_transcript = None; + } + if text.len() > MAX_TRANSCRIPT_BYTES { + let mut end = MAX_TRANSCRIPT_BYTES; + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); + } + if role == "user" { + self.realtime_conversation.latest_voice_input_fingerprint = voice_input_fingerprint; + } + if !text.trim().is_empty() { + while self.realtime_conversation.accepted_transcripts.len() + >= MAX_PENDING_TRANSCRIPT_CELLS + { + self.realtime_conversation.accepted_transcripts.pop_front(); + } + self.realtime_conversation + .accepted_transcripts + .push_back(RealtimeTranscriptRecord { + role: role.clone(), + text: text.clone(), + complete: true, + }); + while self.realtime_conversation.pending_history_cells.len() + >= MAX_PENDING_TRANSCRIPT_CELLS + { + self.realtime_conversation.pending_history_cells.pop_front(); + tracing::warn!("dropping the oldest deferred voice transcript"); + } + let cell = self.realtime_transcript_history_cell(&role, &text); + self.realtime_conversation + .pending_history_cells + .push_back(cell); + self.flush_realtime_transcript_history(); + } + } + + fn finish_realtime_partial_transcripts(&mut self) { + if let Some((role, text)) = self.realtime_conversation.interleaved_transcript.take() { + self.on_realtime_transcript_done(role, text); + } + if let Some(role) = self.realtime_conversation.transcript_role.clone() { + let text = std::mem::take(&mut self.realtime_conversation.transcript); + self.on_realtime_transcript_done(role, text); + } + } + + fn realtime_transcript_history_cell(&self, role: &str, text: &str) -> Box { + if role == "user" { + Box::new(history_cell::new_user_prompt( + text.to_string(), + Vec::new(), + Vec::new(), + Vec::new(), + )) + } else { + Box::new( + history_cell::AgentMarkdownCell::new_with_inline_visualizations( + text.to_string(), + &self.config.cwd, + /*inline_visualization_context*/ None, + ), + ) + } + } + + pub(super) fn flush_realtime_transcript_history(&mut self) { + if self.stream_controller.is_some() + || self.plan_stream_controller.is_some() + || self.pending_stream_consolidations > 0 + { + return; + } + while let Some(cell) = self.realtime_conversation.pending_history_cells.pop_front() { + self.add_boxed_history(cell); + } + } + + pub(crate) fn on_realtime_error(&mut self, message: String) { + if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive { + return; + } + self.stop_realtime_conversation(); + self.add_error_message(message); + } + + pub(super) fn on_realtime_conversation_closed(&mut self, reason: Option) { + if self.realtime_conversation.phase == RealtimeConversationPhase::Inactive { + if self.realtime_conversation.recover_late_transcripts { + self.finish_realtime_partial_transcripts(); + self.realtime_conversation.recover_late_transcripts = false; + } + return; + } + let retry_after_stop = + self.realtime_conversation.startup_retry == StartupRetry::WaitingForStop; + let retry_after_early_close = self.realtime_conversation.phase + == RealtimeConversationPhase::Starting + && self.realtime_conversation.startup_retry == StartupRetry::Available + && reason.as_deref() == Some("transport_closed"); + if self.realtime_conversation.phase == RealtimeConversationPhase::Stopping + && reason.as_deref() != Some("requested") + { + return; + } + // A transport close may arrive without final transcript events. Recover + // answers before flushing partial captions, which do not prove delivery. + self.restore_all_undelivered_realtime_speech(); + self.finish_realtime_partial_transcripts(); + let muted = self.realtime_conversation.microphone_muted; + let thread_id = self.realtime_conversation.thread_id; + self.reset_realtime_conversation(); + if (retry_after_stop || retry_after_early_close) + && let Some(thread_id) = thread_id.filter(|id| Some(*id) == self.thread_id()) + { + // The old backend is closed. A late peer result belongs to its attempt ID. + self.realtime_conversation.startup_retry = StartupRetry::Used; + self.realtime_conversation.microphone_muted = muted; + if retry_after_early_close { + self.add_info_message( + "Voice connection closed during startup. Retrying once.".into(), + /*hint*/ None, + ); + } + self.start_realtime_conversation(thread_id); + return; + } + if let Some(reason) = reason + && reason != "error" + { + self.add_info_message( + format!("Voice conversation ended: {reason}"), + /*hint*/ None, + ); + } + self.request_redraw(); + } + + pub(crate) fn reset_realtime_conversation(&mut self) -> Option { + self.restore_all_undelivered_realtime_speech(); + // A delegated item is hidden until the turn completes so it can be spoken. + // If voice ends first, let the later turn completion render it normally. + if self + .transcript + .last_completed_agent_message + .as_ref() + .is_some_and(|completed_item| { + matches!( + self.realtime_conversation.agent_items.get(completed_item), + Some(RealtimeAgentItemOrigin::Delegated { + completed: true, + .. + }) + ) + }) + { + self.transcript.last_completed_agent_message = None; + } + // The helper may already have exited while app-server still owns the + // voice session. An acknowledged backend start still needs a stop RPC. + let backend_thread_id = if self.realtime_conversation.handle.is_some() + || self.realtime_conversation.backend_started + { + self.realtime_conversation.thread_id + } else { + None + }; + if let Some(abort) = self.realtime_conversation.startup_abort.take() { + abort.abort(); + } + if let Some(handle) = self.realtime_conversation.handle.take() { + handle.close(); + } + // Direct resets (for example, switching to a resumed thread) do not + // pass through the normal stop/close transcript flush. + let partials = [ + self.realtime_conversation.interleaved_transcript.take(), + self.realtime_conversation + .transcript_role + .take() + .map(|role| { + ( + role, + std::mem::take(&mut self.realtime_conversation.transcript), + ) + }), + ]; + for (role, text) in partials.into_iter().flatten() { + if text.trim().is_empty() { + continue; + } + if self.realtime_conversation.accepted_transcripts.len() >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.accepted_transcripts.pop_front(); + } + if self.realtime_conversation.pending_history_cells.len() >= MAX_REPLAY_TRANSCRIPT_CELLS + { + self.realtime_conversation.pending_history_cells.pop_front(); + } + let cell = self.realtime_transcript_history_cell(&role, &text); + self.realtime_conversation + .pending_history_cells + .push_back(cell); + self.realtime_conversation + .accepted_transcripts + .push_back(RealtimeTranscriptRecord { + role, + text, + complete: true, + }); + } + let pending_history_cells = + std::mem::take(&mut self.realtime_conversation.pending_history_cells); + let accepted_transcripts = + std::mem::take(&mut self.realtime_conversation.accepted_transcripts); + let delegated_reasoning_turns = + std::mem::take(&mut self.realtime_conversation.delegated_reasoning_turns); + let had_live_transcript = self.realtime_conversation.live_transcript_cell.is_some(); + self.realtime_conversation = RealtimeConversationUiState { + attempt_id: self.realtime_conversation.attempt_id, + pending_history_cells, + accepted_transcripts, + delegated_reasoning_turns, + ..RealtimeConversationUiState::default() + }; + if had_live_transcript { + self.bump_active_cell_revision(); + } + self.set_footer_hint_override(/*items*/ None); + self.flush_realtime_transcript_history(); + backend_thread_id + } + + pub(super) fn refresh_realtime_microphone_level(&mut self) { + if !matches!( + self.realtime_conversation.phase, + RealtimeConversationPhase::Starting | RealtimeConversationPhase::Active + ) { + return; + } + let Some(handle) = self.realtime_conversation.handle.as_ref() else { + return; + }; + if let Some(error) = handle.take_error() { + self.on_realtime_error(format!("Voice conversation failed: {error}")); + return; + } + if self.realtime_conversation.phase == RealtimeConversationPhase::Starting { + self.frame_requester + .schedule_frame_in(MICROPHONE_METER_INTERVAL); + return; + } + let microphone_level = if self.realtime_conversation.microphone_muted { + 0 + } else { + audio_meter_level(handle.take_microphone_peak()) + }; + let speaker_level = audio_meter_level(handle.take_speaker_peak()); + let mut changed = self.realtime_conversation.microphone_level != microphone_level + || self.realtime_conversation.speaker_level != speaker_level; + if speaker_level > 0 { + self.realtime_conversation.speaker_active_until = + Some(Instant::now() + SPEAKER_ACTIVITY_HOLD); + } else { + changed |= self + .realtime_conversation + .speaker_active_until + .take_if(|deadline| *deadline <= Instant::now()) + .is_some(); + } + self.realtime_conversation.microphone_level = microphone_level; + self.realtime_conversation.speaker_level = speaker_level; + if changed { + self.update_realtime_footer(); + } + self.frame_requester + .schedule_frame_in(MICROPHONE_METER_INTERVAL); + } + + fn update_realtime_footer(&mut self) { + let status = if self.realtime_conversation.phase == RealtimeConversationPhase::Starting { + "◌ connecting" + } else if self.realtime_conversation.microphone_muted { + "◌ muted" + } else if self.realtime_conversation.speaker_level > 0 + || self + .realtime_conversation + .speaker_active_until + .is_some_and(|deadline| deadline > Instant::now()) + { + "● speaking" + } else { + "● listening" + }; + let mut hints = vec![ + ("voice".to_string(), status.to_string()), + ("/voice".to_string(), "stop".to_string()), + ]; + if self.realtime_conversation.phase == RealtimeConversationPhase::Active { + let microphone = if self.realtime_conversation.microphone_muted { + "off".to_string() + } else { + audio_meter_bar(self.realtime_conversation.microphone_level) + }; + let speaker = audio_meter_bar(self.realtime_conversation.speaker_level); + hints.push(("mic".to_string(), format!("{microphone} codex {speaker}"))); + } + hints.push(("/voice".to_string(), "mute".to_string())); + self.set_footer_hint_override(Some(hints)); + } +} + +fn audio_meter_level(peak: u16) -> usize { + usize::from(peak.saturating_sub(AUDIO_METER_NOISE_FLOOR)) + .saturating_mul(AUDIO_METER_SEGMENTS) + .div_ceil(usize::from( + AUDIO_METER_FULL_SCALE - AUDIO_METER_NOISE_FLOOR, + )) + .min(AUDIO_METER_SEGMENTS) +} + +fn audio_meter_bar(level: usize) -> String { + let level = level.min(AUDIO_METER_SEGMENTS); + format!( + "{}{}", + "█".repeat(level), + "░".repeat(AUDIO_METER_SEGMENTS - level) + ) +} diff --git a/codex-rs/tui/src/chatwidget/realtime_tests.rs b/codex-rs/tui/src/chatwidget/realtime_tests.rs new file mode 100644 index 0000000000..8c194c47a2 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/realtime_tests.rs @@ -0,0 +1,107 @@ +//! Focused visual coverage and shared setup for voice behavior tests. + +use super::RealtimeConversationPhase; +use crate::app_event::AppEvent; +use crate::chatwidget::ChatWidget; +use crate::chatwidget::tests::make_chatwidget_manual_with_sender; +use crate::chatwidget::tests::render_bottom_popup; +use codex_protocol::ThreadId; + +pub(crate) fn activate_voice_for_thread(chat: &mut ChatWidget, thread_id: ThreadId) { + chat.thread_id = Some(thread_id); + chat.realtime_conversation.phase = RealtimeConversationPhase::Active; + chat.realtime_conversation.thread_id = Some(thread_id); + chat.realtime_conversation.backend_started = true; + chat.realtime_conversation.latest_input_was_voice = true; +} + +#[tokio::test] +async fn enabling_voice_on_an_open_thread_snapshots_the_new_thread_notice() { + let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await; + chat.set_feature_enabled( + codex_features::Feature::RealtimeConversation, + /*enabled*/ true, + ); + let rendered = std::iter::from_fn(|| events.try_recv().ok()) + .filter_map(|event| match event { + AppEvent::InsertHistoryCell(cell) => Some( + cell.display_lines(/*width*/ 80) + .into_iter() + .map(|line| line.to_string()) + .collect::>() + .join("\n"), + ), + _ => None, + }) + .collect::>() + .join("\n"); + insta::assert_snapshot!(rendered, @"• Voice conversations will be available in new threads."); +} + +#[tokio::test] +async fn voice_footer_renders_the_main_conversation_states() { + let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await; + let mut states = Vec::new(); + + for (label, phase, muted, level, speaker_level, role, transcript) in [ + ( + "connecting", + RealtimeConversationPhase::Starting, + false, + 0, + 0, + None, + "", + ), + ( + "listening", + RealtimeConversationPhase::Active, + false, + 0, + 0, + None, + "", + ), + ( + "speaking", + RealtimeConversationPhase::Active, + false, + 4, + 5, + None, + "", + ), + ( + "muted", + RealtimeConversationPhase::Active, + true, + 4, + 4, + None, + "", + ), + ( + "transcript", + RealtimeConversationPhase::Active, + false, + 2, + 2, + Some("assistant"), + "Hello there", + ), + ] { + chat.realtime_conversation.phase = phase; + chat.realtime_conversation.microphone_muted = muted; + chat.realtime_conversation.microphone_level = level; + chat.realtime_conversation.speaker_level = speaker_level; + chat.realtime_conversation.transcript_role = role.map(str::to_string); + chat.realtime_conversation.transcript = transcript.to_string(); + chat.update_realtime_footer(); + states.push(format!( + "{label}:\n{}", + render_bottom_popup(&chat, /*width*/ 80) + )); + } + + insta::assert_snapshot!(states.join("\n\n")); +} diff --git a/codex-rs/tui/src/chatwidget/reconnect.rs b/codex-rs/tui/src/chatwidget/reconnect.rs index 1de24dcb74..63d5d28f59 100644 --- a/codex-rs/tui/src/chatwidget/reconnect.rs +++ b/codex-rs/tui/src/chatwidget/reconnect.rs @@ -4,6 +4,9 @@ use super::*; impl ChatWidget { pub(crate) fn pause_for_disconnect(&mut self) { + // The app-server transport can fail while the separate WebRTC helper + // still sends microphone audio. Retire local media before showing offline UI. + let _ = self.reset_realtime_conversation(); if let Some(questions) = &mut self.bottom_pane.questions { questions.delivery_enabled = false; } diff --git a/codex-rs/tui/src/chatwidget/rendering.rs b/codex-rs/tui/src/chatwidget/rendering.rs index 61a5403d27..8163ae2549 100644 --- a/codex-rs/tui/src/chatwidget/rendering.rs +++ b/codex-rs/tui/src/chatwidget/rendering.rs @@ -158,6 +158,17 @@ impl ChatWidget { }; let mut flex = FlexRenderable::new(); flex.push(/*flex*/ 1, active_cell_renderable); + if let Some(cell) = self.realtime_conversation.live_transcript_cell.as_ref() { + flex.push( + /*flex*/ 1, + RenderableItem::Owned(Box::new(TranscriptAreaRenderable { + child: cell.as_ref(), + top: 1, + right: active_cell_right_reserve, + persistent_layout: None, + })), + ); + } if let Some(cell) = self.pending_token_activity_output() { flex.push( /*flex*/ 1, diff --git a/codex-rs/tui/src/chatwidget/replay.rs b/codex-rs/tui/src/chatwidget/replay.rs index 18ad2d01f3..a233ce20e0 100644 --- a/codex-rs/tui/src/chatwidget/replay.rs +++ b/codex-rs/tui/src/chatwidget/replay.rs @@ -46,13 +46,31 @@ impl ChatWidget { completed_at, duration_ms, } = turn; + let delegated = items.iter().any(|item| { + matches!(item, ThreadItem::UserMessage { content, .. } + if realtime::realtime_delegation_input(content).is_some()) + }); if matches!(status, TurnStatus::InProgress) { + if delegated { + self.remember_realtime_delegated_reasoning_turn(&turn_id); + } self.warning_display_state.startup_complete = true; self.turn_lifecycle.last_turn_id = Some(turn_id.clone()); self.last_non_retry_error = None; self.on_task_started(); } + let mut replaying_delegation = false; for item in items { + if matches!(&item, ThreadItem::UserMessage { content, .. } + if realtime::realtime_delegation_input(content).is_some()) + { + replaying_delegation = true; + } + // Voice can steer a typed turn already in progress. Its earlier + // commentary and reasoning still belong to the typed request. + if replaying_delegation && realtime::is_private_realtime_agent_item(&item) { + continue; + } if hidden_nested_review_turn && matches!(item, ThreadItem::UserMessage { .. }) { continue; } @@ -122,7 +140,12 @@ impl ChatWidget { ThreadItem::UserMessage { content, client_id, .. } => { - self.on_committed_user_message(&content, client_id.as_deref(), from_replay); + self.on_committed_user_message( + &content, + client_id.as_deref(), + from_replay, + &turn_id, + ); } ThreadItem::AgentMessage { id, @@ -133,6 +156,20 @@ impl ChatWidget { questions, .. } => { + if self.complete_realtime_delegated_agent_item( + &turn_id, + &ThreadItem::AgentMessage { + id: id.clone(), + text: text.clone(), + phase: phase.clone(), + memory_citation: memory_citation.clone(), + delivery, + questions: questions.clone(), + }, + from_replay, + ) { + return; + } self.on_agent_message_item_completed( AgentMessageItem { id, diff --git a/codex-rs/tui/src/chatwidget/session_flow.rs b/codex-rs/tui/src/chatwidget/session_flow.rs index c29a574643..a24e3695f0 100644 --- a/codex-rs/tui/src/chatwidget/session_flow.rs +++ b/codex-rs/tui/src/chatwidget/session_flow.rs @@ -24,6 +24,11 @@ impl ChatWidget { let connector_scope_changed = previous_thread_id != Some(session.thread_id) || self.config.cwd.as_path() != session.cwd.as_path(); self.thread_id = Some(session.thread_id); + self.realtime_conversation_available_for_thread = + self.config.features.enabled(Feature::RealtimeConversation) + && codex_realtime_webrtc::RealtimeWebrtcSession::is_supported(); + self.bottom_pane + .set_voice_command_enabled(self.realtime_conversation_available_for_thread); self.bottom_pane .set_queue_submissions(/*queue_submissions*/ false); if previous_thread_id != self.thread_id { diff --git a/codex-rs/tui/src/chatwidget/settings.rs b/codex-rs/tui/src/chatwidget/settings.rs index 224feab67a..c630d43f41 100644 --- a/codex-rs/tui/src/chatwidget/settings.rs +++ b/codex-rs/tui/src/chatwidget/settings.rs @@ -100,6 +100,21 @@ impl ChatWidget { self.update_collaboration_mode_indicator(); } } + if feature == Feature::RealtimeConversation && !enabled { + self.realtime_conversation_available_for_thread = false; + self.bottom_pane + .set_voice_command_enabled(/*enabled*/ false); + self.stop_realtime_conversation(); + } + if feature == Feature::RealtimeConversation + && enabled + && !self.realtime_conversation_available_for_thread + { + self.add_info_message( + "Voice conversations will be available in new threads.".into(), + /*hint*/ None, + ); + } if feature == Feature::MentionsV2 { self.sync_mentions_v2_enabled(); } diff --git a/codex-rs/tui/src/chatwidget/slash_dispatch.rs b/codex-rs/tui/src/chatwidget/slash_dispatch.rs index b8db07ee74..453aad4022 100644 --- a/codex-rs/tui/src/chatwidget/slash_dispatch.rs +++ b/codex-rs/tui/src/chatwidget/slash_dispatch.rs @@ -333,6 +333,9 @@ impl ChatWidget { ); } } + SlashCommand::Voice => { + self.toggle_realtime_conversation(); + } SlashCommand::Side | SlashCommand::Btw => { self.request_empty_side_conversation(cmd); } @@ -757,6 +760,11 @@ impl ChatWidget { } } } + SlashCommand::Voice => match trimmed.to_ascii_lowercase().as_str() { + "mute" => self.toggle_realtime_microphone(), + "stop" => self.stop_realtime_conversation(), + _ => self.add_error_message("Usage: /voice [mute|stop]".to_string()), + }, SlashCommand::Ide => { self.handle_ide_command_args(trimmed); } @@ -1144,6 +1152,7 @@ impl ChatWidget { goal_command_enabled: self.config.features.enabled(Feature::Goals), service_tier_commands_enabled: self.fast_mode_enabled(), personality_command_enabled: self.config.features.enabled(Feature::Personality), + voice_command_enabled: self.realtime_conversation_available_for_thread, worktrees_enabled: self.config.features.enabled(Feature::Worktrees) && self.local_worktree_operations, allow_elevate_sandbox, @@ -1183,6 +1192,7 @@ impl ChatWidget { | SlashCommand::Diff | SlashCommand::App | SlashCommand::Rename + | SlashCommand::Voice | SlashCommand::Recap | SlashCommand::TestApproval => QueueDrain::Continue, SlashCommand::Cd => match self.thread_id { diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap new file mode 100644 index 0000000000..cb205e97df --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime__tests__voice_footer_renders_the_main_conversation_states.snap @@ -0,0 +1,29 @@ +--- +source: tui/src/chatwidget/realtime_tests.rs +assertion_line: 176 +expression: "states.join(\"\\n\\n\")" +--- +connecting: +› Ask Codex to do anything + + voice ◌ connecting /voice stop /voice mute + +listening: +› Ask Codex to do anything + + voice ● listening /voice stop mic ░░░░░ codex ░░░░░ /voice mute + +speaking: +› Ask Codex to do anything + + voice ● speaking /voice stop mic ████░ codex █████ /voice mute + +muted: +› Ask Codex to do anything + + voice ◌ muted /voice stop mic off codex ████░ /voice mute + +transcript: +› Ask Codex to do anything + + voice ● speaking /voice stop mic ██░░░ codex ██░░░ /voice mute diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap new file mode 100644 index 0000000000..8ff8096740 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__experimental_features_available_popup.snap @@ -0,0 +1,16 @@ +--- +source: tui/src/chatwidget/tests/popups_and_settings.rs +assertion_line: 3142 +expression: popup +--- + Experimental features + Checked features are configured on. Some experimental features take effect + only in new tasks or after restarting the Codex server. + +› [ ] Network proxy Apply network proxy restrictions to + sandboxed sessions that already have + network access. + [ ] Prevent sleep while running Keep your computer awake while Codex is + running a thread. + + Press space to select or enter to save diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap new file mode 100644 index 0000000000..61937b7844 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__tests__voice_live_transcript_and_stream.snap @@ -0,0 +1,16 @@ +--- +source: tui/src/chatwidget/tests/status_and_layout.rs +assertion_line: 34 +expression: rendered +--- +" " +"• Agent answer arriving " +" " +" " +"› pick a number " +" " +" " +" " +"› Ask Codex to do anything " +" " +" voice ● listening /voice stop mic ░░░░░ codex ░░░░░ " diff --git a/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs b/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs index 908af55709..4516460017 100644 --- a/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs +++ b/codex-rs/tui/src/chatwidget/tests/misalignment_policy_tests.rs @@ -1,8 +1,51 @@ use super::*; +use crate::chatwidget::realtime::tests::activate_voice_for_thread; use pretty_assertions::assert_eq; const ERROR_MESSAGE: &str = "Responses API returned misalignment_policy_violation"; +#[tokio::test] +async fn misalignment_precaution_retires_voice_and_keeps_late_turn_blocked() { + let (mut chat, _events, mut ops) = make_chatwidget_manual(/*model_override*/ None).await; + let thread_id = ThreadId::new(); + chat.thread_id = Some(thread_id); + handle_turn_started(&mut chat, "unsafe-turn"); + activate_voice_for_thread(&mut chat, thread_id); + + handle_error( + &mut chat, + ERROR_MESSAGE, + Some(CodexErrorInfo::MisalignmentPolicyViolation), + ); + + assert!(!chat.realtime_conversation_is_running()); + assert_matches!( + ops.try_recv(), + Ok(Op::RealtimeConversationStop { thread_id: stopped }) if stopped == thread_id + ); + assert!(ops.try_recv().is_err()); + assert!(chat.has_misalignment_policy_violation()); + assert!(!chat.bottom_pane.composer_input_enabled()); + assert!(chat.submit_op(Op::RealtimeConversationStop { thread_id })); + assert_matches!(ops.try_recv(), Ok(Op::RealtimeConversationStop { .. })); + + // A handoff already in flight may still produce a TurnStarted notification. + // It must not dismiss the precaution or restart local capture. + handle_turn_started(&mut chat, "late-voice-turn"); + assert!(chat.has_misalignment_policy_violation()); + assert!(!chat.bottom_pane.composer_input_enabled()); + chat.toggle_realtime_conversation(); + assert!(!chat.realtime_conversation_is_running()); + assert!(ops.try_recv().is_err()); + + chat.clear_misalignment_for_new_turn( + "acknowledged-turn", + MisalignmentTurnSource::AcknowledgedContinuation, + ); + assert!(!chat.has_misalignment_policy_violation()); + assert!(chat.bottom_pane.composer_input_enabled()); +} + #[tokio::test] async fn misalignment_policy_failure_stops_the_thread_and_renders_once() { let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index e7617cc3c9..1260a61fbc 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -3175,6 +3175,35 @@ async fn experimental_popup_loading_snapshot() { assert!(rx.try_recv().is_err()); } +#[tokio::test] +async fn experimental_popup_available_snapshot() { + let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.thread_id = Some(ThreadId::new()); + chat.open_experimental_popup(); + let AppEvent::FetchExperimentalFeatures { response_tx, .. } = rx.try_recv().unwrap() else { + panic!("expected experimental discovery request"); + }; + let features = [ + ("network_proxy", "Network proxy", "Apply network proxy restrictions to sandboxed sessions that already have network access."), + ("prevent_idle_sleep", "Prevent sleep while running", "Keep your computer awake while Codex is running a thread."), + ] + .into_iter() + .map(|(name, display_name, description)| codex_app_server_protocol::ExperimentalFeature { + name: name.to_string(), + stage: codex_app_server_protocol::ExperimentalFeatureStage::Beta, + display_name: Some(display_name.to_string()), + description: Some(description.to_string()), + announcement: None, + enabled: false, + default_enabled: false, + }) + .collect(); + response_tx.send(Ok(features)).unwrap(); + chat.pre_draw_tick(); + let popup = render_bottom_popup(&chat, /*width*/ 80); + assert_chatwidget_snapshot!("experimental_features_available_popup", popup); +} + #[tokio::test] async fn feature_enable_prompts_snapshot() { let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; diff --git a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs index 4ae4c253f3..8de9141a6f 100644 --- a/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs +++ b/codex-rs/tui/src/chatwidget/tests/status_and_layout.rs @@ -3,12 +3,37 @@ use crate::bottom_pane::goal_status_indicator_line; use crate::chatwidget::ThreadUsageOutcome; use crate::chatwidget::rate_limits::NUDGE_MODEL_SLUG; use crate::chatwidget::rate_limits::get_limits_duration; +use crate::chatwidget::realtime::tests::activate_voice_for_thread; use codex_app_server_protocol::SpendControlLimitSnapshot; use codex_app_server_protocol::ThreadUsage; use pretty_assertions::assert_eq; +use ratatui::Terminal; use ratatui::backend::TestBackend; use serial_test::serial; +#[tokio::test] +async fn voice_live_transcript_renders_beside_the_streamed_cell() { + let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await; + chat.config.animations = false; + activate_voice_for_thread(&mut chat, ThreadId::new()); + chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new( + vec![Line::from("Agent answer arriving").into()], + /*is_first_line*/ true, + ))); + chat.on_realtime_transcript_delta("user".into(), "pick a number".into()); + + let width = 60; + let height = chat.desired_height(width); + let mut terminal = Terminal::new(TestBackend::new(width, height)).expect("create terminal"); + terminal + .draw(|frame| chat.render(frame.area(), frame.buffer_mut())) + .expect("render live voice transcript"); + let rendered = normalized_backend_snapshot(terminal.backend()); + assert!(rendered.contains("Agent answer arriving"), "{rendered}"); + assert!(rendered.contains("pick a number"), "{rendered}"); + assert_chatwidget_snapshot!("voice_live_transcript_and_stream", rendered); +} + fn enable_test_ambient_pet(chat: &mut ChatWidget) { chat.set_pet_image_support_for_tests(crate::pets::PetImageSupport::Supported( crate::pets::ImageProtocol::Kitty, diff --git a/codex-rs/tui/src/chatwidget/turn_runtime.rs b/codex-rs/tui/src/chatwidget/turn_runtime.rs index 18f51fff29..e52dc9118b 100644 --- a/codex-rs/tui/src/chatwidget/turn_runtime.rs +++ b/codex-rs/tui/src/chatwidget/turn_runtime.rs @@ -208,7 +208,14 @@ impl ChatWidget { .current_goal_status .as_ref() .is_some_and(GoalStatusState::is_active); - if !follow_up_started && !active_goal_continuing { + if !follow_up_started + && !active_goal_continuing + && !self + .turn_lifecycle + .last_turn_id + .as_deref() + .is_some_and(|turn_id| self.should_hide_realtime_delegation(turn_id)) + { self.notify(Notification::AgentTurnComplete { response: notification_response, }); diff --git a/codex-rs/tui/src/slash_command.rs b/codex-rs/tui/src/slash_command.rs index ca83861bdb..93feadc318 100644 --- a/codex-rs/tui/src/slash_command.rs +++ b/codex-rs/tui/src/slash_command.rs @@ -41,6 +41,7 @@ pub enum SlashCommand { Compact, Recap, Plan, + Voice, Goal, Agents, Side, @@ -130,6 +131,7 @@ impl SlashCommand { } SlashCommand::Personality => "choose a communication style for Codex", SlashCommand::Plan => "switch to Plan mode", + SlashCommand::Voice => "start or stop a live voice conversation", SlashCommand::Goal => "set or view the goal for a long-running task", SlashCommand::Agents => "view and switch between all active agent sessions", SlashCommand::MultiAgents => "switch between this session's subagents", @@ -172,6 +174,7 @@ impl SlashCommand { | SlashCommand::Fork | SlashCommand::Plan | SlashCommand::Goal + | SlashCommand::Voice | SlashCommand::Ide | SlashCommand::Keymap | SlashCommand::Mcp @@ -250,6 +253,7 @@ impl SlashCommand { | SlashCommand::Stop | SlashCommand::App | SlashCommand::Goal + | SlashCommand::Voice | SlashCommand::Mcp | SlashCommand::Apps | SlashCommand::Plugins @@ -274,6 +278,7 @@ impl SlashCommand { SlashCommand::SandboxReadRoot => cfg!(target_os = "windows"), SlashCommand::Copy => !cfg!(target_os = "android"), SlashCommand::App => cfg!(any(target_os = "macos", target_os = "windows")), + SlashCommand::Voice => true, SlashCommand::Rollout | SlashCommand::TestApproval => cfg!(debug_assertions), _ => true, } diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs index 3fc237eb89..20c252eb21 100644 --- a/codex-rs/voice-host/src/main.rs +++ b/codex-rs/voice-host/src/main.rs @@ -176,9 +176,15 @@ fn run( let Some(peer) = transport.as_ref() else { return Err(io::Error::other("voice transport not started")); }; - executor + let outcome = executor .block_on(peer.apply_answer(sdp.into_sdp())) .map_err(io::Error::other)?; + if outcome == transport::AnswerOutcome::TimedOut { + // The client reaps this helper before considering a fresh negotiation. + output.write_all(&encode_frame(&Message::TransportTimedOut {})?)?; + output.flush()?; + return Ok(()); + } answered = true; Message::TransportReady {} } @@ -207,6 +213,7 @@ fn run( | Message::ApplyAnswer { .. } | Message::Offer { .. } | Message::TransportReady {} + | Message::TransportTimedOut {} | Message::OpenDevices {} | Message::DevicesOpened {} | Message::AudioControlsApplied {} diff --git a/codex-rs/voice-host/src/transport.rs b/codex-rs/voice-host/src/transport.rs index 3b0646576d..e3769093a2 100644 --- a/codex-rs/voice-host/src/transport.rs +++ b/codex-rs/voice-host/src/transport.rs @@ -59,6 +59,12 @@ impl PeerConnectionEventHandler for Events { } } +#[derive(Debug, PartialEq)] +pub(crate) enum AnswerOutcome { + Ready, + TimedOut, +} + pub(crate) struct Transport { pub(crate) incoming: crate::incoming::Incoming, pub(crate) audio: crate::audio_track::AudioTrack, @@ -162,7 +168,7 @@ impl Transport { .map_err(|_| "timed out gathering voice candidates")? } - pub(crate) async fn apply_answer(&self, sdp: String) -> Result<()> { + pub(crate) async fn apply_answer(&self, sdp: String) -> Result { let answer = RTCSessionDescription::answer(sdp).map_err(|_| "invalid voice answer")?; let parsed = answer.unmarshal().map_err(|_| "invalid voice answer")?; let mut seen = HashSet::new(); @@ -213,7 +219,9 @@ impl Transport { .map_err(|_| "voice event channel closed") }) .await - .map_err(|_| "timed out connecting voice peer")? + .map_or(Ok(AnswerOutcome::TimedOut), |result| { + result.map(|()| AnswerOutcome::Ready) + }) } pub(crate) async fn close(&mut self) -> Result<()> { diff --git a/codex-rs/voice-host/src/transport_tests.rs b/codex-rs/voice-host/src/transport_tests.rs index 20eda40ad9..3646959820 100644 --- a/codex-rs/voice-host/src/transport_tests.rs +++ b/codex-rs/voice-host/src/transport_tests.rs @@ -86,10 +86,11 @@ async fn check_negotiation(runtime: Arc) { for _ in candidates.len()..MAX_REMOTE_CANDIDATES { answer.push_str(&format!("{}\r\n", candidates[0])); } - local + let outcome = local .apply_answer(answer) .await .unwrap_or_else(|error| panic!("tcp={tcp}: {error}")); + assert_eq!(outcome, super::AnswerOutcome::Ready); let channel = channels.recv().await.unwrap(); assert_eq!( (