mirror of
https://github.com/openai/codex.git
synced 2026-10-27 18:48:37 +00:00
Stabilize realtime voice meter sampling across redraws (#43695)
## Why Footer updates and transcript animation can trigger redraws before more audio arrives, consuming peaks too early and adding spurious quiet samples. Old peaks also linger in the meter after a channel becomes quiet. ## What changed - Sample microphone and speaker peaks at most once every 100 ms, preserving accumulated peaks on early redraws and scheduling the next sampling frame. - Clear each channel's meter history on its first quiet sample without clearing the other channel. ## Testing Add regression tests for early redraw scheduling, peak preservation, independent channel silence, delayed sampling, and sampling after a conversation reset. Snapshot the meters at wide and narrow widths and after each channel becomes quiet. GitOrigin-RevId: d7b116af2c099a2eaeb6db0d66bca93b230db970
This commit is contained in:
committed by
copyberry
parent
45305dd229
commit
9a3af22d01
@@ -145,6 +145,7 @@ pub(super) struct RealtimeConversationUiState {
|
||||
microphone_history: VoiceAmplitudeHistory,
|
||||
speaker_history: VoiceAmplitudeHistory,
|
||||
audio_meter_history: VecDeque<(u8, u8)>,
|
||||
next_audio_meter_sample_at: Option<Instant>,
|
||||
speaker_active_until: Option<Instant>,
|
||||
interruption_acknowledged_until: Option<Instant>,
|
||||
speaker_suppression_generation: Option<u64>,
|
||||
|
||||
@@ -123,12 +123,33 @@ impl ChatWidget {
|
||||
.schedule_frame_in(MICROPHONE_METER_INTERVAL);
|
||||
return;
|
||||
}
|
||||
let handle = handle.clone();
|
||||
self.refresh_realtime_audio_meters(Instant::now(), || {
|
||||
(handle.take_microphone_peak(), handle.take_speaker_peak())
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn refresh_realtime_audio_meters(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
take_peaks: impl FnOnce() -> (u16, u16),
|
||||
) {
|
||||
// Redraws are not sampling ticks: updating the footer itself requests a redraw.
|
||||
// Leave the peaks untouched until a full sampling interval has elapsed.
|
||||
if let Some(next_sample_at) = self.realtime_conversation.next_audio_meter_sample_at
|
||||
&& now < next_sample_at
|
||||
{
|
||||
self.frame_requester.schedule_frame_in(next_sample_at - now);
|
||||
return;
|
||||
}
|
||||
self.realtime_conversation.next_audio_meter_sample_at =
|
||||
Some(now + MICROPHONE_METER_INTERVAL);
|
||||
let (microphone_peak, speaker_peak) = take_peaks();
|
||||
let microphone_peak = if self.realtime_conversation.microphone_muted {
|
||||
0
|
||||
} else {
|
||||
handle.take_microphone_peak()
|
||||
microphone_peak
|
||||
};
|
||||
let speaker_peak = handle.take_speaker_peak();
|
||||
let microphone_level = audio_meter_level(microphone_peak);
|
||||
let speaker_level = audio_meter_level(speaker_peak);
|
||||
let microphone_intensity = audio_meter_intensity(microphone_peak);
|
||||
@@ -140,19 +161,18 @@ impl ChatWidget {
|
||||
|| self.realtime_conversation.microphone_intensity != microphone_intensity
|
||||
|| self.realtime_conversation.speaker_intensity != speaker_intensity;
|
||||
if speaker_level > 0 {
|
||||
self.realtime_conversation.speaker_active_until =
|
||||
Some(Instant::now() + SPEAKER_ACTIVITY_HOLD);
|
||||
self.realtime_conversation.speaker_active_until = Some(now + SPEAKER_ACTIVITY_HOLD);
|
||||
} else {
|
||||
changed |= self
|
||||
.realtime_conversation
|
||||
.speaker_active_until
|
||||
.take_if(|deadline| *deadline <= Instant::now())
|
||||
.take_if(|deadline| *deadline <= now)
|
||||
.is_some();
|
||||
}
|
||||
changed |= self
|
||||
.realtime_conversation
|
||||
.interruption_acknowledged_until
|
||||
.take_if(|deadline| *deadline <= Instant::now())
|
||||
.take_if(|deadline| *deadline <= now)
|
||||
.is_some();
|
||||
self.realtime_conversation.microphone_level = microphone_level;
|
||||
self.realtime_conversation.speaker_level = speaker_level;
|
||||
@@ -169,6 +189,15 @@ impl ChatWidget {
|
||||
.audio_meter_history
|
||||
.iter()
|
||||
.any(|(microphone, speaker)| *microphone > 0 || *speaker > 0);
|
||||
// Release a quiet channel instead of waiting for old peaks to scroll out.
|
||||
for (microphone, speaker) in &mut self.realtime_conversation.audio_meter_history {
|
||||
if microphone_intensity == 0 {
|
||||
*microphone = 0;
|
||||
}
|
||||
if speaker_intensity == 0 {
|
||||
*speaker = 0;
|
||||
}
|
||||
}
|
||||
if self.realtime_conversation.audio_meter_history.len() >= MAX_REALTIME_AUDIO_METER_FRAMES {
|
||||
self.realtime_conversation.audio_meter_history.pop_front();
|
||||
}
|
||||
|
||||
@@ -207,6 +207,99 @@ async fn voice_preserves_the_normal_composer_prompt() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn voice_meters_do_not_sample_again_on_early_redraws() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
activate_voice(&mut chat);
|
||||
let now = std::time::Instant::now();
|
||||
let peaks = std::cell::Cell::new((0, 0));
|
||||
for elapsed_ms in [0, 100, 200, 300, 400, 500] {
|
||||
let sample_at = now + std::time::Duration::from_millis(elapsed_ms);
|
||||
peaks.set((8192, 4096));
|
||||
chat.refresh_realtime_audio_meters(sample_at, || peaks.replace((0, 0)));
|
||||
// Footer updates and transcript animation can redraw before more audio arrives.
|
||||
let (frame_requester, mut frame_requests) = crate::tui::FrameRequester::test_channel();
|
||||
chat.frame_requester = frame_requester;
|
||||
let before_redraw = std::time::Instant::now();
|
||||
chat.refresh_realtime_audio_meters(sample_at + std::time::Duration::from_millis(8), || {
|
||||
peaks.replace((0, 0))
|
||||
});
|
||||
let after_redraw = std::time::Instant::now();
|
||||
let remaining = std::time::Duration::from_millis(92);
|
||||
let deadline = frame_requests
|
||||
.try_recv()
|
||||
.expect("early redraw re-arms sampling");
|
||||
assert!((before_redraw + remaining..=after_redraw + remaining).contains(&deadline));
|
||||
assert!(frame_requests.try_recv().is_err());
|
||||
}
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.audio_meter_history,
|
||||
std::collections::VecDeque::from([(255, 119); 6])
|
||||
);
|
||||
let render_meter = |chat: &ChatWidget, width| {
|
||||
render_bottom_popup(chat, width)
|
||||
.lines()
|
||||
.find(|line| line.contains("mic "))
|
||||
.unwrap()
|
||||
.trim()
|
||||
.to_string()
|
||||
};
|
||||
let mut meters = [80, 30].map(|width| render_meter(&chat, width)).to_vec();
|
||||
|
||||
// An early redraw must also leave newly accumulated peaks for the next sample.
|
||||
peaks.set((4096, 8192));
|
||||
chat.refresh_realtime_audio_meters(now + std::time::Duration::from_millis(599), || {
|
||||
peaks.replace((0, 0))
|
||||
});
|
||||
chat.refresh_realtime_audio_meters(now + std::time::Duration::from_millis(600), || {
|
||||
peaks.replace((0, 0))
|
||||
});
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.audio_meter_history,
|
||||
std::collections::VecDeque::from([
|
||||
(255, 119),
|
||||
(255, 119),
|
||||
(255, 119),
|
||||
(255, 119),
|
||||
(255, 119),
|
||||
(255, 119),
|
||||
(119, 255)
|
||||
])
|
||||
);
|
||||
// Each channel settles on its first quiet sample, without clearing the other one.
|
||||
for (elapsed_ms, peaks) in [(700, (0, 8192)), (800, (8192, 0))] {
|
||||
chat.refresh_realtime_audio_meters(
|
||||
now + std::time::Duration::from_millis(elapsed_ms),
|
||||
|| peaks,
|
||||
);
|
||||
meters.push(render_meter(&chat, /*width*/ 80));
|
||||
}
|
||||
insta::assert_snapshot!(meters.join("\n"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn voice_meters_preserve_silence_and_restart_sampling_after_reset() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
activate_voice(&mut chat);
|
||||
let now = std::time::Instant::now();
|
||||
chat.refresh_realtime_audio_meters(now, || (8192, 4096));
|
||||
// A delayed draw adds one real sample, not synthetic catch-up bars.
|
||||
chat.refresh_realtime_audio_meters(now + std::time::Duration::from_secs(1), || (0, 0));
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.audio_meter_history,
|
||||
std::collections::VecDeque::from([(0, 0), (0, 0)])
|
||||
);
|
||||
chat.reset_realtime_conversation();
|
||||
activate_voice(&mut chat);
|
||||
chat.refresh_realtime_audio_meters(now + std::time::Duration::from_millis(1001), || {
|
||||
(4096, 8192)
|
||||
});
|
||||
assert_eq!(
|
||||
chat.realtime_conversation.audio_meter_history,
|
||||
std::collections::VecDeque::from([(119, 255)])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn voice_footer_renders_the_main_conversation_states() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
---
|
||||
source: tui/src/chatwidget/realtime/recording_controls_tests.rs
|
||||
expression: "meters.join(\"\\n\")"
|
||||
---
|
||||
mic ██████ codex ▅▅▅▅▅▅
|
||||
mic ██████ codex ▅▅▅▅▅▅
|
||||
mic ▁▁▁▁▁▁ codex ▅▅▅▅██
|
||||
mic ▁▁▁▁▁█ codex ▁▁▁▁▁▁
|
||||
Reference in New Issue
Block a user