From fc81a7154bc6b53f0f2e2b9ecdb33504e0b38f5d Mon Sep 17 00:00:00 2001 From: Benjamin Carlsson Date: Wed, 30 Sep 2026 01:45:47 +0000 Subject: [PATCH] Add local audio device selection to TUI voice settings (#49437) ## Why Voice conversations previously opened the system-default microphone and speaker. Users need to choose which local devices to use, including when connected to a remote app server. ## What changed - Add input and output device pickers alongside voice selection, with back navigation and a System default option. - Persist selections as `audio.microphone` and `audio.speaker` on the TUI's machine, excluding project configuration layers. Apply them to the next voice conversation. - Enumerate devices without opening audio streams and pass the selected names to the voice helper at startup. - Disable ambiguous device names in the picker and use the same eligibility filtering for discovery and named-device startup. ## Testing Add tests for device preference persistence across project and remote transitions, picker navigation and duplicate names, discovery without capture, and forwarding selected devices to the helper. GitOrigin-RevId: d48fb3a09fa9b982a408f5a7c96bdd81776af21d --- codex-rs/realtime-webrtc/src/client.rs | 24 ++- codex-rs/realtime-webrtc/src/lib.rs | 3 + codex-rs/realtime-webrtc/src/protocol.rs | 29 +++- codex-rs/realtime-webrtc/src/session.rs | 11 +- codex-rs/realtime-webrtc/tests/common/mod.rs | 17 +- .../realtime-webrtc/tests/session_actor.rs | 49 +++++- codex-rs/tui/src/app/event_dispatch.rs | 16 +- codex-rs/tui/src/app/realtime_settings.rs | 113 +++++++++++- .../tui/src/app/tests/realtime_requests.rs | 127 +++++++++++++- ...s__remote_voice_picker_server_default.snap | 1 + codex-rs/tui/src/app_event.rs | 14 ++ codex-rs/tui/src/chatwidget/realtime.rs | 11 +- .../tui/src/chatwidget/realtime_settings.rs | 141 ++++++++++++++- .../src/chatwidget/realtime_settings_tests.rs | 161 +++++++++++++++++- ..._tests__voice_ambiguous_input_devices.snap | 14 ++ ...altime_settings__tests__voice_choices.snap | 18 ++ ...sts__voice_multichannel_input_devices.snap | 14 ++ ...e_multichannel_selected_input_devices.snap | 14 ++ ...settings__tests__voice_output_devices.snap | 14 ++ ...ltime_settings__tests__voice_settings.snap | 15 +- ...__tests__voice_settings_input_channel.snap | 13 ++ ..._tests__voice_settings_server_catalog.snap | 1 + ...gs__tests__voice_stereo_input_devices.snap | 14 ++ codex-rs/tui/src/local_settings.rs | 13 ++ codex-rs/voice-host/src/devices.rs | 86 +++++++++- codex-rs/voice-host/src/devices_tests.rs | 33 ++++ .../voice-host/src/devices_unavailable.rs | 18 +- codex-rs/voice-host/src/main.rs | 12 +- 28 files changed, 941 insertions(+), 55 deletions(-) create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_ambiguous_input_devices.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_choices.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_input_devices.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_selected_input_devices.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_output_devices.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_input_channel.snap create mode 100644 codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_stereo_input_devices.snap diff --git a/codex-rs/realtime-webrtc/src/client.rs b/codex-rs/realtime-webrtc/src/client.rs index ed2f50e37f..d6ef6af207 100644 --- a/codex-rs/realtime-webrtc/src/client.rs +++ b/codex-rs/realtime-webrtc/src/client.rs @@ -12,6 +12,9 @@ use codex_utils_pty::SpawnedProcess; use tokio::sync::oneshot; use tokio::time::timeout; +use crate::AudioDevice; +use crate::AudioDeviceKind; +use crate::AudioDeviceSelection; use crate::HelperExitStage; use crate::Message; use crate::encode_frame; @@ -63,12 +66,27 @@ pub struct VoiceHost { impl VoiceHost { /// Open local devices only after answer negotiation. They initially remain muted/suppressed. - pub async fn open_devices(mut self) -> Result { - self.exchange(Message::OpenDevices {}, Message::DevicesOpened {}, DEADLINE) - .await?; + pub async fn open_devices(mut self, selection: AudioDeviceSelection) -> Result { + self.exchange( + Message::OpenDevices { selection }, + Message::DevicesOpened {}, + DEADLINE, + ) + .await?; Ok(self) } + /// Enumerate local devices without opening streams. + pub async fn list_devices(&mut self, kind: AudioDeviceKind) -> Result> { + match self + .request(Message::ListDevices { kind }, DEADLINE) + .await? + { + Message::DeviceList { devices } => Ok(devices), + _ => anyhow::bail!("unexpected voice helper response"), + } + } + /// Acknowledgement follows invalidation of the helper's previous capture/render generations. pub async fn set_audio_controls(&mut self, controls: crate::AudioControls) -> Result<()> { self.exchange( diff --git a/codex-rs/realtime-webrtc/src/lib.rs b/codex-rs/realtime-webrtc/src/lib.rs index d4776cfcdd..988365628c 100644 --- a/codex-rs/realtime-webrtc/src/lib.rs +++ b/codex-rs/realtime-webrtc/src/lib.rs @@ -10,6 +10,9 @@ pub use client::ConnectionError; pub use client::VoiceHost; pub use helper_exit::HelperExitStage; pub use protocol::AudioControls; +pub use protocol::AudioDevice; +pub use protocol::AudioDeviceKind; +pub use protocol::AudioDeviceSelection; pub use protocol::AudioState; pub use protocol::MAX_FRAME_BYTES; pub use protocol::Message; diff --git a/codex-rs/realtime-webrtc/src/protocol.rs b/codex-rs/realtime-webrtc/src/protocol.rs index 2859aaea87..6c067243b4 100644 --- a/codex-rs/realtime-webrtc/src/protocol.rs +++ b/codex-rs/realtime-webrtc/src/protocol.rs @@ -23,6 +23,31 @@ pub struct AudioState { pub speaker_peak: u16, } +/// Direction of a local audio device, shared by enumeration and capture/playback selection. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum AudioDeviceKind { + Input, + Output, +} + +/// Local device preferences fixed for the lifetime of a voice conversation. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AudioDeviceSelection { + pub microphone: Option, + pub speaker: Option, +} + +/// Local device metadata; enumeration never opens streams. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AudioDevice { + pub name: String, + pub channels: u16, + pub is_default: bool, +} + /// SDP contains ICE credentials. Bound it at construction and never expose it in diagnostics. #[derive(Deserialize, PartialEq, Serialize)] #[serde(try_from = "String")] @@ -82,7 +107,9 @@ pub enum Message { ApplyAnswer { sdp: SessionDescription }, TransportReady {}, TransportTimedOut {}, - OpenDevices {}, + ListDevices { kind: AudioDeviceKind }, + DeviceList { devices: Vec }, + OpenDevices { selection: AudioDeviceSelection }, DevicesOpened {}, SetAudioControls { controls: AudioControls }, AudioControlsApplied {}, diff --git a/codex-rs/realtime-webrtc/src/session.rs b/codex-rs/realtime-webrtc/src/session.rs index fee0c39669..badbf41e38 100644 --- a/codex-rs/realtime-webrtc/src/session.rs +++ b/codex-rs/realtime-webrtc/src/session.rs @@ -16,6 +16,7 @@ use futures::future::Abortable; use tokio::sync::mpsc; use crate::AudioControls; +use crate::AudioDeviceSelection; use crate::ConnectionError; use crate::SessionDescription; use crate::VoiceHost; @@ -93,7 +94,10 @@ impl RealtimeWebrtcSession { } /// Called off the UI thread. Cancellation also owns startup before a handle is returned. - pub fn start(abort: AbortRegistration) -> Result { + pub fn start( + abort: AbortRegistration, + selection: AudioDeviceSelection, + ) -> Result { let package = codex_install_context::InstallContext::current() .package_layout .clone() @@ -147,7 +151,7 @@ impl RealtimeWebrtcSession { offer .send(Ok(sdp.into_sdp())) .map_err(|_| anyhow::anyhow!("voice startup cancelled"))?; - run(host, receiver, &state, &controls).await + run(host, receiver, &state, &controls, selection).await }; let result = runtime.block_on(Abortable::new(Abortable::new(task, abort), stopped)); if let Ok(Ok(Err(error))) = result { @@ -266,6 +270,7 @@ async fn run( mut commands: mpsc::Receiver, state: &State, controls: &Mutex, + selection: AudioDeviceSelection, ) -> Result<()> { let mut connected = false; let mut poll = tokio::time::interval(Duration::from_millis(/*millis*/ 50)); @@ -277,7 +282,7 @@ async fn run( Some(Command::Answer(sdp, complete)) if !connected => { let startup = async { let mut host = report_failure(ConnectionError::Transport, host.apply_answer(sdp).await)?; - host = report_failure(ConnectionError::AudioDevices, host.open_devices().await)?; + host = report_failure(ConnectionError::AudioDevices, host.open_devices(selection.clone()).await)?; let applied = startup_controls(&mut commands, controls, |initial| { host.begin_audio_controls(initial) }); diff --git a/codex-rs/realtime-webrtc/tests/common/mod.rs b/codex-rs/realtime-webrtc/tests/common/mod.rs index 63f8884b48..137012d28c 100644 --- a/codex-rs/realtime-webrtc/tests/common/mod.rs +++ b/codex-rs/realtime-webrtc/tests/common/mod.rs @@ -148,6 +148,17 @@ fn helper() -> Result<()> { stage = Stage::Initialize; Message::Ready {} } + Message::ListDevices { kind } if stage == Stage::Initialize => Message::DeviceList { + devices: vec![codex_realtime_webrtc::AudioDevice { + name: match kind { + codex_realtime_webrtc::AudioDeviceKind::Input => "Interface", + codex_realtime_webrtc::AudioDeviceKind::Output => "Headphones", + } + .into(), + channels: 16, + is_default: true, + }], + }, Message::InitializeRuntime {} if stage == Stage::Initialize => { fs::write(root.join("initializing"), [])?; if root.join("hold-initialization").exists() { @@ -178,7 +189,11 @@ fn helper() -> Result<()> { stage = Stage::Devices; Message::TransportReady {} } - Message::OpenDevices {} if stage == Stage::Devices => { + Message::OpenDevices { selection } if stage == Stage::Devices => { + fs::write( + root.join("input-selection"), + serde_json::to_vec(&selection)?, + )?; if root.join("fail-devices").exists() { anyhow::bail!("synthetic private device diagnostic"); } diff --git a/codex-rs/realtime-webrtc/tests/session_actor.rs b/codex-rs/realtime-webrtc/tests/session_actor.rs index ded26288ce..524f5ded14 100644 --- a/codex-rs/realtime-webrtc/tests/session_actor.rs +++ b/codex-rs/realtime-webrtc/tests/session_actor.rs @@ -4,6 +4,8 @@ mod common; use anyhow::Result; use codex_realtime_webrtc::AudioControls; +use codex_realtime_webrtc::AudioDeviceKind; +use codex_realtime_webrtc::AudioDeviceSelection; use codex_realtime_webrtc::RealtimeWebrtcSession; use futures::future::AbortHandle; use pretty_assertions::assert_eq; @@ -24,7 +26,13 @@ fn startup_controls_meters_and_helper_loss() -> Result<()> { assert!(RealtimeWebrtcSession::is_supported()); } let (_abort, registration) = AbortHandle::new_pair(); - let started = RealtimeWebrtcSession::start(registration)?; + let started = RealtimeWebrtcSession::start( + registration, + AudioDeviceSelection { + microphone: Some("Zen Go Synergy Core Playback".into()), + speaker: Some("Headphones".into()), + }, + )?; assert_eq!(started.offer_sdp, "synthetic-offer"); let handle = started.handle; handle.set_microphone_muted(/*muted*/ true)?; @@ -34,6 +42,10 @@ fn startup_controls_meters_and_helper_loss() -> Result<()> { handle.set_speaker_suppressed(/*suppressed*/ true); fs::write(root.join("release"), [])?; answer.join().expect("answer thread")?; + assert_eq!( + serde_json::from_slice::(&fs::read(root.join("input-selection"))?)?, + serde_json::json!({"microphone": "Zen Go Synergy Core Playback", "speaker": "Headphones"}) + ); let initial: Vec = serde_json::from_slice(&fs::read(root.join("controls"))?)?; assert_eq!( initial, @@ -84,7 +96,10 @@ fn external_cancellation_interrupts_startup() -> Result<()> { let (abort, registration) = AbortHandle::new_pair(); let (result, received) = std::sync::mpsc::sync_channel(/*bound*/ 1); let startup = thread::spawn(move || { - let _ = result.send(RealtimeWebrtcSession::start(registration)); + let _ = result.send(RealtimeWebrtcSession::start( + registration, + AudioDeviceSelection::default(), + )); }); common::wait_for(|| root.join("initializing").exists())?; abort.abort(); @@ -106,7 +121,7 @@ fn last_owner_drop_reaps_helper() -> Result<()> { return Ok(()); }; let (_abort, registration) = AbortHandle::new_pair(); - let started = RealtimeWebrtcSession::start(registration)?; + let started = RealtimeWebrtcSession::start(registration, AudioDeviceSelection::default())?; drop(started); common::wait_for_helper_reaped(&root) } @@ -121,7 +136,7 @@ fn device_failure_reaches_startup_completion_without_duplicate_error() -> Result fs::write(root.join("fail-devices"), [])?; fs::write(root.join("release"), [])?; let (_abort, registration) = AbortHandle::new_pair(); - let started = RealtimeWebrtcSession::start(registration)?; + let started = RealtimeWebrtcSession::start(registration, AudioDeviceSelection::default())?; assert_eq!( started.handle.apply_answer_sdp("synthetic-answer".into()), Err(codex_realtime_webrtc::ConnectionError::AudioDevices) @@ -141,7 +156,8 @@ fn runtime_failure_reaches_offer_caller_without_native_error_text() -> Result<() }; fs::write(root.join("fail-initialization"), [])?; let (_abort, registration) = AbortHandle::new_pair(); - let error = RealtimeWebrtcSession::start(registration).unwrap_err(); + let error = + RealtimeWebrtcSession::start(registration, AudioDeviceSelection::default()).unwrap_err(); assert_eq!( error.downcast_ref::(), Some(&codex_realtime_webrtc::ConnectionError::RuntimeInitialization) @@ -154,3 +170,26 @@ fn runtime_failure_reaches_offer_caller_without_native_error_text() -> Result<() common::wait_for_helper_reaped(&root)?; Ok(()) } + +#[tokio::test] +async fn device_discovery_does_not_start_capture() -> Result<()> { + let Some(root) = common::package("device_discovery_does_not_start_capture")? else { + return Ok(()); + }; + let package = codex_install_context::InstallContext::current() + .package_layout + .as_ref() + .unwrap(); + let mut host = codex_realtime_webrtc::VoiceHost::connect(package, common::BUILD_COMMIT).await?; + assert_eq!( + host.list_devices(AudioDeviceKind::Input).await?, + vec![codex_realtime_webrtc::AudioDevice { + name: "Interface".into(), + channels: 16, + is_default: true, + }] + ); + host.close().await?; + assert!(!root.join("initializing").exists()); + Ok(()) +} diff --git a/codex-rs/tui/src/app/event_dispatch.rs b/codex-rs/tui/src/app/event_dispatch.rs index f8fd11d57b..da47c0f7e9 100644 --- a/codex-rs/tui/src/app/event_dispatch.rs +++ b/codex-rs/tui/src/app/event_dispatch.rs @@ -2451,8 +2451,20 @@ impl App { } self.chat_widget.on_plugin_mentions_loaded(plugins); } - AppEvent::OpenRealtimeSettings => { - self.open_realtime_settings(app_server).await; + AppEvent::OpenRealtimeSettings => self.chat_widget.open_realtime_settings(), + AppEvent::OpenRealtimeSoundDevices => self.chat_widget.open_realtime_sound_devices(), + AppEvent::OpenRealtimeVoices => self.open_realtime_voices(app_server).await, + AppEvent::OpenRealtimeDevicePicker { kind } => self.list_realtime_devices(kind), + AppEvent::RealtimeDevicesListed { origin, kind, result } => { + if origin == self.active_thread_id { + match result { + Ok(devices) => self.chat_widget.open_realtime_device_picker(kind, devices), + Err(error) => self.chat_widget.add_error_message(error), + } + } + } + AppEvent::PersistRealtimeDevice { kind, name } => { + self.persist_realtime_device(kind, name).await; } AppEvent::PersistRealtimeVoiceSelection { voice } => { self.persist_realtime_voice(app_server, voice).await; diff --git a/codex-rs/tui/src/app/realtime_settings.rs b/codex-rs/tui/src/app/realtime_settings.rs index 2f31148ee6..92e301bd13 100644 --- a/codex-rs/tui/src/app/realtime_settings.rs +++ b/codex-rs/tui/src/app/realtime_settings.rs @@ -1,12 +1,14 @@ -//! Resolve and persist voice preferences through the owning server before updating the UI. +//! Persist server voice preferences and machine-local audio selections before updating the UI. use super::*; +use crate::legacy_core::config::edit::ConfigEdit; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ThreadRealtimeListVoicesParams; use codex_app_server_protocol::ThreadRealtimeListVoicesResponse; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; +use codex_realtime_webrtc::AudioDeviceKind; use uuid::Uuid; /// Unknown voices belong to newer servers; omit the override instead of blocking audio. @@ -29,6 +31,111 @@ fn configured_voice( } impl App { + pub(super) async fn persist_realtime_device( + &mut self, + kind: AudioDeviceKind, + name: Option, + ) { + let key = match kind { + AudioDeviceKind::Input => "microphone", + AudioDeviceKind::Output => "speaker", + }; + self.persist_realtime_audio(key, name.map(Into::into)).await; + self.list_realtime_devices(kind); + } + + async fn persist_realtime_audio(&mut self, key: &str, value: Option) { + // Audio runs on the TUI's machine, even when the app server is remote. + let segments = vec!["audio".to_string(), key.to_string()]; + let edits = vec![match &value { + Some(toml::Value::String(value)) => ConfigEdit::SetPath { + segments, + value: value.clone().into(), + }, + Some(_) => unreachable!("audio device settings are names"), + None => ConfigEdit::ClearPath { segments }, + }]; + if let Err(error) = + ConfigEditsBuilder::for_config_path(self.local_settings.user_config_path.as_path()) + .with_edits(edits) + .apply() + .await + { + self.chat_widget + .add_error_message(format!("Failed to save audio setting: {error}")); + return; + } + match crate::legacy_core::config::load_config_toml_with_layer_stack( + self.local_settings.codex_home.as_path(), + /*cwd*/ None, + self.cli_kv_overrides.clone(), + codex_config::ConfigLoadOptions { + loader_overrides: self.loader_overrides.clone(), + cloud_config_bundle: self.cloud_config_bundle.clone(), + ..Default::default() + }, + ) + .await + { + Ok(config) => { + self.config.config_layer_stack = self + .config + .config_layer_stack + .with_user_layer_from(&config.config_layer_stack); + let audio = config.config_toml.audio.unwrap_or_default(); + let effective = toml::Value::try_from(&audio) + .ok() + .and_then(|audio| audio.get(key).cloned()); + self.local_settings.audio = Ok(audio); + self.chat_widget.local_settings.audio = self.local_settings.audio.clone(); + if effective == value { + self.chat_widget.add_info_message( + "Audio setting saved. Applies to your next voice conversation.".to_string(), + /*hint*/ None, + ); + } else { + self.chat_widget.add_error_message( + "Audio setting was saved but is overridden by another configuration layer." + .to_string(), + ); + } + } + Err(error) => self.chat_widget.add_error_message(format!( + "Audio setting was saved, but effective settings could not be read: {error}" + )), + } + } + + pub(super) fn list_realtime_devices(&self, kind: AudioDeviceKind) { + let tx = self.app_event_tx.clone(); + let origin = self.active_thread_id; + tokio::spawn(async move { + let result = async { + let package = codex_install_context::InstallContext::current() + .package_layout + .as_ref() + .ok_or_else(|| anyhow::anyhow!("voice package unavailable"))?; + let mut host = codex_realtime_webrtc::VoiceHost::connect( + package, + codex_build_info::BuildInfo::get().build_commit(), + ) + .await?; + let result = host.list_devices(kind).await; + host.close().await?; + result + } + .await; + tx.send(AppEvent::RealtimeDevicesListed { + origin, + kind, + result: result.map_err(|_| { + "Could not list audio devices. Check your audio device and voice package." + .to_owned() + }), + }); + }); + } + pub(super) async fn realtime_voices( &self, app_server: &AppServerSession, @@ -63,7 +170,7 @@ impl App { .map(Option::flatten) } - pub(super) async fn open_realtime_settings(&mut self, app_server: &AppServerSession) { + pub(super) async fn open_realtime_voices(&mut self, app_server: &AppServerSession) { let voices = match self.realtime_voices(app_server).await { Ok(voices) => voices, Err(error) => { @@ -73,7 +180,7 @@ impl App { } }; match self.effective_realtime_voice(app_server, &voices).await { - Ok(voice) => self.chat_widget.open_realtime_settings(voice, voices), + Ok(voice) => self.chat_widget.open_realtime_voices(voice, voices), Err(error) => self .chat_widget .add_error_message(format!("Failed to read voice settings: {error}")), diff --git a/codex-rs/tui/src/app/tests/realtime_requests.rs b/codex-rs/tui/src/app/tests/realtime_requests.rs index 16c81ceecf..57bcbf2ea6 100644 --- a/codex-rs/tui/src/app/tests/realtime_requests.rs +++ b/codex-rs/tui/src/app/tests/realtime_requests.rs @@ -7,6 +7,7 @@ use crate::app::tests::session_lifecycle_requests::start_recording_realtime_spee use crate::app::tests::session_lifecycle_requests::start_recording_remote_app_server; use crate::chatwidget::commit_realtime_history_events; use crate::chatwidget::tests::make_chatwidget_manual_with_sender; +use crate::local_settings::LocalSettings; use codex_app_server_protocol::ItemCompletedNotification; use codex_app_server_protocol::ItemStartedNotification; use codex_app_server_protocol::ServerNotification; @@ -1832,7 +1833,7 @@ async fn embedded_voice_settings_follow_project_after_thread_switch() -> Result< ), ); Box::pin(app.select_agent_thread(&mut tui, &mut server, thread_id)).await?; - app.open_realtime_settings(&server).await; + app.open_realtime_voices(&server).await; let popup = crate::chatwidget::tests::helpers::render_bottom_popup( &app.chat_widget, /*width*/ 80, @@ -1872,7 +1873,7 @@ async fn remote_voice_picker_reads_server_preference() -> Result<()> { app.chat_widget .set_realtime_voice(Some(codex_protocol::protocol::RealtimeVoice::Maple)); let (server, requests, proxy) = start_recording_remote_app_server(&app.config).await?; - app.open_realtime_settings(&server).await; + app.open_realtime_voices(&server).await; assert_eq!( recorded_params(&requests, "thread/realtime/listVoices").len(), 1 @@ -1906,7 +1907,7 @@ async fn remote_voice_catalog_success_and_failure() -> Result<()> { ) .await?; while events.try_recv().is_ok() {} - app.open_realtime_settings(&server).await; + app.open_realtime_voices(&server).await; let popup = crate::chatwidget::tests::helpers::render_bottom_popup( &app.chat_widget, /*width*/ 80, @@ -2078,3 +2079,123 @@ async fn overridden_voice_save_keeps_effective_voice() -> Result<()> { } Ok(()) } + +#[tokio::test] +async fn audio_devices_are_persisted() -> Result<()> { + use codex_realtime_webrtc::AudioDeviceKind; + let mut app = make_test_app().await; + app.persist_realtime_device(AudioDeviceKind::Output, Some("Headphones".into())) + .await; + app.persist_realtime_device(AudioDeviceKind::Input, Some("Interface".into())) + .await; + let expected = codex_config::config_toml::RealtimeAudioToml { + microphone: Some("Interface".into()), + speaker: Some("Headphones".into()), + }; + assert_eq!(app.local_settings.audio.as_ref().unwrap(), &expected); + assert_eq!(LocalSettings::from(&app.config).audio, Ok(expected)); + app.persist_realtime_device(AudioDeviceKind::Input, /*name*/ None) + .await; + app.persist_realtime_device(AudioDeviceKind::Output, /*name*/ None) + .await; + assert_eq!(app.local_settings.audio, Ok(Default::default())); + assert_eq!( + app.chat_widget.local_settings.audio, + app.local_settings.audio + ); + Ok(()) +} + +#[tokio::test] +async fn audio_device_is_machine_local_across_project_and_remote_transitions() -> Result<()> { + let mut app = make_test_app().await; + let project = tempfile::tempdir()?; + let remote = tempfile::tempdir()?; + std::fs::create_dir_all(project.path().join(".codex"))?; + std::fs::write( + project.path().join(".codex/config.toml"), + "default_permissions = \"work\"\n[audio]\nmicrophone = \"Project mic\"\n[permissions.work]\nextends = \":workspace\"\n", + )?; + crate::legacy_core::config::set_project_trust_level( + &app.config.codex_home, + project.path(), + codex_protocol::config_types::TrustLevel::Trusted, + ) + .map_err(|error| color_eyre::eyre::eyre!(error.to_string()))?; + app.cli_kv_overrides = vec![("default_permissions".into(), "work".into())]; + app.config = app.rebuild_config_for_cwd(project.path().into()).await?; + app.app_server_target = AppServerTarget::Remote { + endpoint: crate::resolve_remote_addr("ws://127.0.0.1:9")?, + }; + app.chat_widget + .handle_thread_session_quiet(test_thread_session( + ThreadId::new(), + remote.path().to_path_buf(), + )); + app.persist_realtime_device( + codex_realtime_webrtc::AudioDeviceKind::Input, + Some("Machine mic".into()), + ) + .await; + app.cli_kv_overrides.clear(); + assert_eq!( + LocalSettings::from(&app.config).audio, + app.local_settings.audio + ); + for cwd in [project.path(), remote.path()] { + let config = app.rebuild_config_for_cwd(cwd.to_path_buf()).await?; + let reloaded = app.local_settings.reloaded(&config); + assert_eq!(reloaded.audio, app.chat_widget.local_settings.audio); + assert_eq!( + reloaded.audio.as_ref().unwrap().microphone.as_deref(), + Some("Machine mic") + ); + } + // A project value must not hide invalid machine audio. + std::fs::write( + app.local_settings.user_config_path.as_path(), + std::fs::read_to_string(app.local_settings.user_config_path.as_path())? + .replace("microphone = \"Machine mic\"", "microphone = 0"), + )?; + let config = app + .rebuild_config_for_cwd(project.path().to_path_buf()) + .await?; + app.chat_widget.local_settings = app.local_settings.reloaded(&config); + assert!(app.chat_widget.local_settings.audio.is_err()); + app.chat_widget.open_realtime_sound_devices(); + assert!(!app.chat_widget.has_active_view()); + + Ok(()) +} + +#[tokio::test] +async fn microphone_discovery_ignores_another_threads_completion() -> Result<()> { + let (mut app, _events, _ops) = make_test_app_with_channels().await; + let (mut server, _requests, proxy) = start_recording_remote_app_server(&app.config).await?; + let mut tui = crate::tui::test_support::make_test_tui()?; + app.active_thread_id = Some(ThreadId::new()); + for origin in [Some(ThreadId::new()), app.active_thread_id] { + let result = Ok(vec![codex_realtime_webrtc::AudioDevice { + name: "Interface".into(), + channels: 16, + is_default: true, + }]); + Box::pin(app.handle_event( + &mut tui, + &mut server, + AppEvent::RealtimeDevicesListed { + origin, + kind: codex_realtime_webrtc::AudioDeviceKind::Input, + result, + }, + )) + .await?; + assert_eq!( + app.chat_widget.has_active_view(), + origin == app.active_thread_id + ); + } + server.shutdown().await?; + proxy.await??; + Ok(()) +} diff --git a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_requests__remote_voice_picker_server_default.snap b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_requests__remote_voice_picker_server_default.snap index 7c6af54461..b4792ba8f4 100644 --- a/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_requests__remote_voice_picker_server_default.snap +++ b/codex-rs/tui/src/app/tests/snapshots/codex_tui__app__tests__realtime_requests__remote_voice_picker_server_default.snap @@ -9,5 +9,6 @@ expression: popup › 1. maple (current) 2. cove 3. juniper + 4. Back enter select · esc back diff --git a/codex-rs/tui/src/app_event.rs b/codex-rs/tui/src/app_event.rs index 9e3a2beb75..dfedc79733 100644 --- a/codex-rs/tui/src/app_event.rs +++ b/codex-rs/tui/src/app_event.rs @@ -1220,6 +1220,20 @@ pub(crate) enum AppEvent { /// Read the owning server preference before showing the voice picker. OpenRealtimeSettings, + OpenRealtimeSoundDevices, + OpenRealtimeVoices, + OpenRealtimeDevicePicker { + kind: codex_realtime_webrtc::AudioDeviceKind, + }, + RealtimeDevicesListed { + origin: Option, + kind: codex_realtime_webrtc::AudioDeviceKind, + result: Result, String>, + }, + PersistRealtimeDevice { + kind: codex_realtime_webrtc::AudioDeviceKind, + name: Option, + }, /// Save the voice for subsequent conversations through the app server. PersistRealtimeVoiceSelection { diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs index ccda3290f4..95e4c90cfb 100644 --- a/codex-rs/tui/src/chatwidget/realtime.rs +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -312,6 +312,9 @@ impl ChatWidget { } fn start_realtime_conversation(&mut self, thread_id: ThreadId) { + let Some(audio) = self.realtime_audio_settings() else { + return; + }; self.realtime_conversation.recover_late_transcripts = false; self.realtime_conversation.attempt_id = NEXT_REALTIME_ATTEMPT_ID.fetch_add(1, Ordering::Relaxed); @@ -322,9 +325,13 @@ impl ChatWidget { self.realtime_conversation.phase = RealtimeConversationPhase::Starting; self.update_realtime_footer(); let app_event_tx = self.app_event_tx.clone(); + let selection = codex_realtime_webrtc::AudioDeviceSelection { + microphone: audio.microphone, + speaker: audio.speaker, + }; std::thread::spawn(move || { - let result = - RealtimeWebrtcSession::start(abort_registration).map_err(|error| error.to_string()); + let result = RealtimeWebrtcSession::start(abort_registration, selection) + .map_err(|error| error.to_string()); app_event_tx.send(AppEvent::RealtimeWebrtcOfferCreated { thread_id, attempt_id, diff --git a/codex-rs/tui/src/chatwidget/realtime_settings.rs b/codex-rs/tui/src/chatwidget/realtime_settings.rs index 3057764af5..e0ff656791 100644 --- a/codex-rs/tui/src/chatwidget/realtime_settings.rs +++ b/codex-rs/tui/src/chatwidget/realtime_settings.rs @@ -1,17 +1,62 @@ -//! Voice selection for subsequent conversations; saving never interrupts live audio. +//! Voice settings hierarchy with parent navigation; selections apply to subsequent conversations. use super::*; use codex_protocol::protocol::RealtimeVoice; use codex_protocol::protocol::RealtimeVoicesList; +use codex_realtime_webrtc::AudioDevice; +use codex_realtime_webrtc::AudioDeviceKind; + +fn back_item(parent: fn() -> AppEvent) -> SelectionItem { + SelectionItem { + name: "Back".to_string(), + actions: vec![Box::new(move |tx| tx.send(parent()))], + dismiss_on_select: true, + ..Default::default() + } +} impl ChatWidget { - pub(crate) fn open_realtime_settings( + pub(super) fn realtime_audio_settings( + &mut self, + ) -> Option { + match self.local_settings.audio.clone() { + Ok(audio) => Some(audio), + Err(error) => { + self.add_error_message(error); + None + } + } + } + + pub(crate) fn open_realtime_settings(&mut self) { + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some("Voice settings".to_string()), + items: vec![ + SelectionItem { + name: "Set sound devices".to_string(), + actions: vec![Box::new(|tx| tx.send(AppEvent::OpenRealtimeSoundDevices))], + dismiss_on_select: true, + ..Default::default() + }, + SelectionItem { + name: "Choose a voice".to_string(), + actions: vec![Box::new(|tx| tx.send(AppEvent::OpenRealtimeVoices))], + dismiss_on_select: true, + ..Default::default() + }, + ], + footer_hint: Some(standard_popup_hint_line()), + ..SelectionViewParams::picker() + }); + } + + pub(crate) fn open_realtime_voices( &mut self, current: Option, voices: RealtimeVoicesList, ) { // The TUI uses V3, which shares the V1 voice catalog. - let items = voices + let mut items: Vec = voices .v1 .into_iter() .map(|voice| SelectionItem { @@ -24,11 +69,101 @@ impl ChatWidget { ..Default::default() }) .collect(); + items.push(back_item(|| AppEvent::OpenRealtimeSettings)); self.bottom_pane.show_selection_view(SelectionViewParams { title: Some("Select voice".to_string()), subtitle: Some("Applies to your next voice conversation.".to_string()), footer_hint: Some(standard_popup_hint_line()), items, + on_cancel: Some(Box::new(|tx| tx.send(AppEvent::OpenRealtimeSettings))), + ..SelectionViewParams::picker() + }); + } + + pub(crate) fn open_realtime_sound_devices(&mut self) { + let Some(audio) = self.realtime_audio_settings() else { + return; + }; + let mut items = Vec::new(); + for (kind, label, selected) in [ + (AudioDeviceKind::Input, "Input device", audio.microphone), + (AudioDeviceKind::Output, "Output device", audio.speaker), + ] { + items.push(SelectionItem { + name: label.into(), + description: Some(selected.unwrap_or_else(|| "System default".into())), + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::OpenRealtimeDevicePicker { kind }) + })], + dismiss_on_select: true, + ..Default::default() + }); + } + items.push(back_item(|| AppEvent::OpenRealtimeSettings)); + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some("Sound devices".to_string()), + subtitle: Some("Applies to your next voice conversation.".into()), + footer_hint: Some(standard_popup_hint_line()), + items, + on_cancel: Some(Box::new(|tx| tx.send(AppEvent::OpenRealtimeSettings))), + ..SelectionViewParams::picker() + }); + } + + pub(crate) fn open_realtime_device_picker( + &mut self, + kind: AudioDeviceKind, + devices: Vec, + ) { + let Some(audio) = self.realtime_audio_settings() else { + return; + }; + let (title, current) = match kind { + AudioDeviceKind::Input => ("Input device", audio.microphone), + AudioDeviceKind::Output => ("Output device", audio.speaker), + }; + let ambiguous: std::collections::HashSet<_> = devices + .iter() + .filter(|device| { + devices + .iter() + .filter(|other| other.name == device.name) + .count() + > 1 + }) + .map(|device| device.name.clone()) + .collect(); + let mut items: Vec = std::iter::once(/*value*/ None) + .chain(devices.into_iter().map(Some)) + .map(|device| { + let name = device.map(|device| device.name); + let is_current = name == current; + let disabled_reason = name + .as_ref() + .filter(|name| ambiguous.contains(*name)) + .map(|_| "Identical device names; use System default.".into()); + SelectionItem { + name: name.clone().unwrap_or_else(|| "System default".into()), + is_current, + disabled_reason, + actions: vec![Box::new(move |tx| { + tx.send(AppEvent::PersistRealtimeDevice { + kind, + name: name.clone(), + }) + })], + dismiss_on_select: true, + ..Default::default() + } + }) + .collect(); + items.push(back_item(|| AppEvent::OpenRealtimeSoundDevices)); + self.bottom_pane.show_selection_view(SelectionViewParams { + title: Some(title.into()), + subtitle: Some("Applies to your next voice conversation.".into()), + items, + footer_hint: Some(standard_popup_hint_line()), + on_cancel: Some(Box::new(|tx| tx.send(AppEvent::OpenRealtimeSoundDevices))), ..SelectionViewParams::picker() }); } diff --git a/codex-rs/tui/src/chatwidget/realtime_settings_tests.rs b/codex-rs/tui/src/chatwidget/realtime_settings_tests.rs index 84657da2bd..846b0f6934 100644 --- a/codex-rs/tui/src/chatwidget/realtime_settings_tests.rs +++ b/codex-rs/tui/src/chatwidget/realtime_settings_tests.rs @@ -1,4 +1,4 @@ -//! Voice picker rendering and selection routing without audio hardware. +//! Voice settings hierarchy, parent navigation and selection routing without audio hardware. use super::*; use crate::chatwidget::tests::make_chatwidget_manual_with_sender; @@ -21,8 +21,18 @@ async fn voice_settings_selects_a_voice_without_starting_audio() { std::iter::from_fn(|| events.try_recv().ok()) .any(|event| matches!(event, AppEvent::OpenRealtimeSettings)) ); - chat.open_realtime_settings(Some(RealtimeVoice::Maple), RealtimeVoicesList::builtin()); + chat.open_realtime_settings(); insta::assert_snapshot!("voice_settings", render_bottom_popup(&chat, /*width*/ 80)); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Down, KeyModifiers::NONE)); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + assert!( + std::iter::from_fn(|| events.try_recv().ok()) + .any(|event| matches!(event, AppEvent::OpenRealtimeVoices)) + ); + chat.open_realtime_voices(Some(RealtimeVoice::Maple), RealtimeVoicesList::builtin()); + insta::assert_snapshot!("voice_choices", render_bottom_popup(&chat, /*width*/ 80)); chat.bottom_pane .handle_key_event(KeyEvent::new(KeyCode::Home, KeyModifiers::NONE)); chat.bottom_pane @@ -57,7 +67,7 @@ async fn saved_voice_confirmation_snapshot() { #[tokio::test] async fn voice_settings_uses_server_catalog() { let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await; - chat.open_realtime_settings( + chat.open_realtime_voices( Some(RealtimeVoice::Cove), RealtimeVoicesList { v1: vec![RealtimeVoice::Cove, RealtimeVoice::Maple], @@ -69,3 +79,148 @@ async fn voice_settings_uses_server_catalog() { render_bottom_popup(&chat, /*width*/ 80) ); } + +#[tokio::test] +async fn sound_devices_picker_routes_selection_without_starting_audio() { + let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await; + chat.local_settings.audio.as_mut().unwrap().microphone = + Some("Zen Go Synergy Core Playback".to_string()); + chat.open_realtime_settings(); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + assert!( + std::iter::from_fn(|| events.try_recv().ok()) + .any(|event| matches!(event, AppEvent::OpenRealtimeSoundDevices)) + ); + chat.open_realtime_sound_devices(); + insta::assert_snapshot!( + "voice_settings_input_channel", + render_bottom_popup(&chat, /*width*/ 90) + ); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Home, KeyModifiers::NONE)); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); + assert!( + std::iter::from_fn(|| events.try_recv().ok()).any(|event| matches!( + event, + AppEvent::OpenRealtimeDevicePicker { + kind: AudioDeviceKind::Input + } + )) + ); + assert!(ops.try_recv().is_err()); + assert!(!chat.realtime_conversation_is_running()); +} + +#[tokio::test] +async fn submenus_return_to_their_parent_with_back_or_escape() { + let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await; + type OpenMenu = fn(&mut ChatWidget); + let menus: [OpenMenu; 2] = [ + |chat| chat.open_realtime_voices(/*current*/ None, RealtimeVoicesList::builtin()), + ChatWidget::open_realtime_sound_devices, + ]; + for open in menus { + for key in [KeyCode::Esc, KeyCode::Enter] { + while events.try_recv().is_ok() {} + open(&mut chat); + if key == KeyCode::Enter { + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::End, KeyModifiers::NONE)); + } + chat.bottom_pane + .handle_key_event(KeyEvent::new(key, KeyModifiers::NONE)); + assert!( + std::iter::from_fn(|| events.try_recv().ok()) + .any(|event| matches!(event, AppEvent::OpenRealtimeSettings)) + ); + } + } + assert!(ops.try_recv().is_err()); + assert!(!chat.realtime_conversation_is_running()); +} + +#[tokio::test] +async fn device_pickers_route_selection() { + let (mut chat, _sender, mut events, mut ops) = make_chatwidget_manual_with_sender().await; + for (kind, channels, microphone, snapshot) in [ + ( + AudioDeviceKind::Input, + 2, + None, + "voice_stereo_input_devices", + ), + ( + AudioDeviceKind::Input, + 4, + None, + "voice_multichannel_input_devices", + ), + ( + AudioDeviceKind::Input, + 4, + Some("Audio interface".into()), + "voice_multichannel_selected_input_devices", + ), + (AudioDeviceKind::Output, 4, None, "voice_output_devices"), + ] { + chat.local_settings.audio.as_mut().unwrap().microphone = microphone; + let device = AudioDevice { + name: "Audio interface".into(), + channels, + is_default: true, + }; + let devices = vec![ + device.clone(), + AudioDevice { + name: "Other device".into(), + channels: 2, + is_default: false, + }, + ]; + chat.open_realtime_device_picker(kind, devices.clone()); + insta::assert_snapshot!(snapshot, render_bottom_popup(&chat, /*width*/ 80)); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + let selected = + std::iter::from_fn(|| events.try_recv().ok()).find_map(|event| match event { + AppEvent::PersistRealtimeDevice { kind, name } => Some((kind, name)), + _ => None, + }); + assert_eq!(selected, Some((kind, Some(device.name.clone())))); + chat.open_realtime_device_picker(kind, devices); + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Char('4'), KeyModifiers::NONE)); + assert!( + std::iter::from_fn(|| events.try_recv().ok()) + .any(|event| matches!(event, AppEvent::OpenRealtimeSoundDevices)) + ); + } + assert!(ops.try_recv().is_err()); + assert!(!chat.realtime_conversation_is_running()); +} + +#[tokio::test] +async fn ambiguous_device_names_cannot_select_the_wrong_device() { + let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await; + let devices = [2, 4] + .map(|channels| AudioDevice { + name: "USB microphone".into(), + channels, + is_default: false, + }) + .to_vec(); + chat.open_realtime_device_picker(AudioDeviceKind::Input, devices); + insta::assert_snapshot!( + "voice_ambiguous_input_devices", + render_bottom_popup(&chat, /*width*/ 80) + ); + while events.try_recv().is_ok() {} + chat.bottom_pane + .handle_key_event(KeyEvent::new(KeyCode::Char('2'), KeyModifiers::NONE)); + assert!( + !std::iter::from_fn(|| events.try_recv().ok()) + .any(|event| matches!(event, AppEvent::PersistRealtimeDevice { .. })) + ); +} diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_ambiguous_input_devices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_ambiguous_input_devices.snap new file mode 100644 index 0000000000..c6ca4fa34c --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_ambiguous_input_devices.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Input device + Applies to your next voice conversation. + + +› 1. System default (current) + USB microphone (disabled) Identical device names; use System default. + USB microphone (disabled) Identical device names; use System default. + 2. Back + + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_choices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_choices.snap new file mode 100644 index 0000000000..531afad01a --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_choices.snap @@ -0,0 +1,18 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Select voice + Applies to your next voice conversation. + + + 1. juniper +› 2. maple (current) + 3. spruce + 4. ember + 5. vale + 6. breeze + 7. arbor + 8. sol +↓ + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_input_devices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_input_devices.snap new file mode 100644 index 0000000000..1ecf807257 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_input_devices.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Input device + Applies to your next voice conversation. + + +› 1. System default (current) + 2. Audio interface + 3. Other device + 4. Back + + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_selected_input_devices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_selected_input_devices.snap new file mode 100644 index 0000000000..919bf16779 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_multichannel_selected_input_devices.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Input device + Applies to your next voice conversation. + + + 1. System default +› 2. Audio interface (current) + 3. Other device + 4. Back + + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_output_devices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_output_devices.snap new file mode 100644 index 0000000000..4f820944cc --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_output_devices.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Output device + Applies to your next voice conversation. + + +› 1. System default (current) + 2. Audio interface + 3. Other device + 4. Back + + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings.snap index 531afad01a..f42fc409e2 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings.snap @@ -2,17 +2,10 @@ source: tui/src/chatwidget/realtime_settings_tests.rs expression: "render_bottom_popup(&chat, 80)" --- - Select voice - Applies to your next voice conversation. + Voice settings - 1. juniper -› 2. maple (current) - 3. spruce - 4. ember - 5. vale - 6. breeze - 7. arbor - 8. sol -↓ +› 1. Set sound devices + 2. Choose a voice + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_input_channel.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_input_channel.snap new file mode 100644 index 0000000000..b5c6594671 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_input_channel.snap @@ -0,0 +1,13 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 90)" +--- + Sound devices + Applies to your next voice conversation. + + +› 1. Input device Zen Go Synergy Core Playback + 2. Output device System default + 3. Back + + enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_server_catalog.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_server_catalog.snap index a78990f3dd..938025893f 100644 --- a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_server_catalog.snap +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_settings_server_catalog.snap @@ -8,5 +8,6 @@ expression: "render_bottom_popup(&chat, 80)" › 1. cove (current) 2. maple + 3. Back enter select · esc back diff --git a/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_stereo_input_devices.snap b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_stereo_input_devices.snap new file mode 100644 index 0000000000..1ecf807257 --- /dev/null +++ b/codex-rs/tui/src/chatwidget/snapshots/codex_tui__chatwidget__realtime_settings__tests__voice_stereo_input_devices.snap @@ -0,0 +1,14 @@ +--- +source: tui/src/chatwidget/realtime_settings_tests.rs +expression: "render_bottom_popup(&chat, 80)" +--- + Input device + Applies to your next voice conversation. + + +› 1. System default (current) + 2. Audio interface + 3. Other device + 4. Back + + enter select · esc back diff --git a/codex-rs/tui/src/local_settings.rs b/codex-rs/tui/src/local_settings.rs index 4eb78a510c..8669f96750 100644 --- a/codex-rs/tui/src/local_settings.rs +++ b/codex-rs/tui/src/local_settings.rs @@ -3,6 +3,7 @@ //! The resolved core config is a temporary input at local load/reload boundaries. Server thread //! responses must never refresh these values; live preference changes belong here. The remaining //! Config-based lifecycle adapters also use this conversion until their interfaces are migrated. +//! Audio preferences exclude project layers so thread cwd cannot route local capture. //! Effective animations also respect the TUI host's launch-time accessibility preference. //! The selected transcript ownership and alternate-screen restrictions survive local reloads. @@ -20,6 +21,7 @@ use codex_utils_absolute_path::AbsolutePathBuf; #[derive(Clone, Debug, PartialEq)] pub(crate) struct LocalSettings { + pub(crate) audio: Result, pub(crate) tui: Tui, pub(crate) transcript_mode: TranscriptMode, pub(crate) history: History, @@ -56,7 +58,18 @@ impl LocalSettings { } else { config.animations }; + let mut audio = toml::Value::Table(Default::default()); + for layer in config.config_layer_stack.layers_low_to_high() { + if !matches!(layer.name, codex_config::ConfigLayerSource::Project { .. }) + && let Some(value) = layer.config.get("audio") + { + codex_config::merge_toml_values(&mut audio, value); + } + } Self { + audio: audio.try_into().map_err(|error: toml::de::Error| { + format!("Invalid machine audio settings: {error}") + }), transcript_mode: TranscriptMode::resolve( config.tui_fullscreen_transcript, config.tui_alternate_screen != codex_config::types::AltScreenMode::Never, diff --git a/codex-rs/voice-host/src/devices.rs b/codex-rs/voice-host/src/devices.rs index a26e4aa7d5..4d5a88c0c2 100644 --- a/codex-rs/voice-host/src/devices.rs +++ b/codex-rs/voice-host/src/devices.rs @@ -23,6 +23,7 @@ use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; +use codex_realtime_webrtc::AudioDeviceKind; use cpal::FromSample; use cpal::Sample; use cpal::SampleFormat; @@ -41,6 +42,77 @@ use buffers::Playback; use buffers::QUEUE_CAPACITY; use playback::PlaybackPort; +fn device( + host: &cpal::Host, + kind: AudioDeviceKind, + name: Option<&str>, +) -> io::Result { + let selected = match (kind, name) { + (AudioDeviceKind::Input, None) => host.default_input_device(), + (AudioDeviceKind::Output, None) => host.default_output_device(), + (kind, Some(name)) => { + let devices = match kind { + AudioDeviceKind::Input => host.input_devices(), + AudioDeviceKind::Output => host.output_devices(), + } + .map_err(io::Error::other)?; + eligible_devices(devices, |device| device_description(device, kind)) + .find(|(_, candidate, _)| candidate == name) + .map(|(device, _, _)| device) + } + }; + selected.ok_or_else(|| io::Error::other("selected audio device unavailable")) +} + +pub(super) fn list_devices( + kind: AudioDeviceKind, +) -> io::Result> { + let host = cpal::default_host(); + let default_id = device(&host, kind, /*name*/ None) + .ok() + .and_then(|device| device.id().ok()); + let devices = match kind { + AudioDeviceKind::Input => host.input_devices(), + AudioDeviceKind::Output => host.output_devices(), + } + .map_err(io::Error::other)?; + Ok( + eligible_devices(devices, |device| device_description(device, kind)) + .map( + |(device, name, channels)| codex_realtime_webrtc::AudioDevice { + name, + channels, + is_default: default_id.is_some() && device.id().ok() == default_id, + }, + ) + .collect(), + ) +} + +fn device_description( + device: &cpal::Device, + kind: AudioDeviceKind, +) -> Option<(String, cpal::SupportedStreamConfig)> { + let config = match kind { + AudioDeviceKind::Input => device.default_input_config(), + AudioDeviceKind::Output => device.default_output_config(), + } + .ok()?; + Some((device.description().ok()?.name().to_owned(), config)) +} + +// Listing and named startup must agree on which endpoints can be selected. +fn eligible_devices( + devices: impl Iterator, + mut describe: impl FnMut(&D) -> Option<(String, cpal::SupportedStreamConfig)>, +) -> impl Iterator { + devices.take(/*n*/ 64).filter_map(move |device| { + let (name, config) = describe(&device)?; + bounded_stream_config(&config).ok()?; + (name.chars().count() <= 256).then(|| (device, name, config.channels())) + }) +} + const MAX_CAPTURE_AGE: Duration = Duration::from_secs(/*secs*/ 1); pub(super) struct Devices { @@ -76,14 +148,14 @@ impl Devices { self.worker.buffers.take_state() } - pub(super) fn open() -> io::Result { + pub(super) fn open(selection: codex_realtime_webrtc::AudioDeviceSelection) -> io::Result { let host = cpal::default_host(); - let input = host - .default_input_device() - .ok_or_else(|| io::Error::other("microphone unavailable"))?; - let output = host - .default_output_device() - .ok_or_else(|| io::Error::other("speaker unavailable"))?; + let input = device( + &host, + AudioDeviceKind::Input, + selection.microphone.as_deref(), + )?; + let output = device(&host, AudioDeviceKind::Output, selection.speaker.as_deref())?; let input_config = input .default_input_config() .map_err(|_| io::Error::other("microphone configuration unavailable"))?; diff --git a/codex-rs/voice-host/src/devices_tests.rs b/codex-rs/voice-host/src/devices_tests.rs index e9f7b40dab..ac1e1c9e47 100644 --- a/codex-rs/voice-host/src/devices_tests.rs +++ b/codex-rs/voice-host/src/devices_tests.rs @@ -423,3 +423,36 @@ fn stream_xruns_recover_without_clearing_device_failure() { handle_stream_error(&buffers, cpal::ErrorKind::Xrun.into()); assert!(buffers.failed.load(Ordering::Acquire)); } + +#[test] +fn named_startup_skips_same_name_devices_omitted_from_discovery() { + let name = "USB interface"; + let supported = |channels| { + cpal::SupportedStreamConfig::new( + channels, + /*sample_rate*/ 48_000, + cpal::SupportedBufferSize::Range { min: 1, max: 4_096 }, + cpal::SampleFormat::F32, + ) + }; + // A missing default configuration and a configuration outside our bounds + // must not shadow the later selectable endpoint with the same name. + let descriptions = [None, Some(supported(64)), Some(supported(2))]; + let candidates = || { + eligible_devices(0..descriptions.len(), |index| { + descriptions[*index].map(|config| (name.into(), config)) + }) + }; + assert_eq!( + candidates().collect::>(), + vec![(2, name.to_string(), 2)] + ); + assert_eq!( + candidates().find(|(_, candidate, _)| candidate == name), + Some((2, name.to_string(), 2)) + ); + assert_eq!( + candidates().find(|(_, candidate, _)| candidate == "Unavailable"), + None + ); +} diff --git a/codex-rs/voice-host/src/devices_unavailable.rs b/codex-rs/voice-host/src/devices_unavailable.rs index 03af795eb1..238b5f385e 100644 --- a/codex-rs/voice-host/src/devices_unavailable.rs +++ b/codex-rs/voice-host/src/devices_unavailable.rs @@ -2,31 +2,39 @@ use std::io; +pub(super) fn list_devices( + _: codex_realtime_webrtc::AudioDeviceKind, +) -> io::Result> { + Err(io::Error::other( + "audio devices unavailable for this helper target", + )) +} + pub(super) struct Devices; impl Devices { pub(super) fn receive(&self, _: crate::incoming::ReceivedRtp) -> io::Result<()> { - Self::open().map(|_| ()) + Self::open(Default::default()).map(|_| ()) } pub(super) fn take_state(&self) -> io::Result { - Self::open().map(|_| codex_realtime_webrtc::AudioState::default()) + Self::open(Default::default()).map(|_| codex_realtime_webrtc::AudioState::default()) } - pub(super) fn open() -> io::Result { + pub(super) fn open(_: codex_realtime_webrtc::AudioDeviceSelection) -> io::Result { Err(io::Error::other( "audio devices unavailable for this helper target", )) } pub(super) fn set_controls(&self, _: codex_realtime_webrtc::AudioControls) -> io::Result<()> { - Self::open().map(|_| ()) + Self::open(Default::default()).map(|_| ()) } pub(super) async fn service( &mut self, _: &mut crate::audio_track::AudioTrack, ) -> io::Result { - Self::open().map(|_| 0) + Self::open(Default::default()).map(|_| 0) } } diff --git a/codex-rs/voice-host/src/main.rs b/codex-rs/voice-host/src/main.rs index 709521aa0e..cd40fddf56 100644 --- a/codex-rs/voice-host/src/main.rs +++ b/codex-rs/voice-host/src/main.rs @@ -168,9 +168,14 @@ fn run( .unwrap_or_default(), } } - Ok(Message::OpenDevices {}) if devices.is_none() && runtime.is_some() && answered => { + Ok(Message::ListDevices { kind }) => Message::DeviceList { + devices: devices::list_devices(kind)?, + }, + Ok(Message::OpenDevices { selection }) + if devices.is_none() && runtime.is_some() && answered => + { phase.set(HelperExitStage::OpenDevices); - devices = Some(devices::Devices::open()?); + devices = Some(devices::Devices::open(selection)?); Message::DevicesOpened {} } Ok(Message::SetAudioControls { controls }) => { @@ -248,7 +253,8 @@ fn run( | Message::Offer { .. } | Message::TransportReady {} | Message::TransportTimedOut {} - | Message::OpenDevices {} + | Message::DeviceList { .. } + | Message::OpenDevices { .. } | Message::DevicesOpened {} | Message::AudioControlsApplied {} | Message::AudioState { .. }