mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Add live WebRTC voice conversations to the TUI (#43581)
## What changed - Add feature-gated `/voice`, `/voice mute`, and `/voice stop` commands with local WebRTC audio and app-server signaling. - Show live transcripts, conversation status, and microphone and speaker levels. Preserve captions and undelivered answers across thread switches. - Speak final answers from voice handoffs while keeping delegated reasoning and commentary hidden and typed answers unspoken. - Retry eligible startup failures once and clean up voice sessions on thread switches and disconnects. - Stop voice and block late handoffs after a misalignment policy violation. Remove realtime event payloads and spoken text from receipt and debug logs. ## Testing Add coverage for voice command mapping, rejected starts, disconnect cleanup, delegated final-answer speech, late handoff rejection, and transcript and footer rendering. GitOrigin-RevId: 11def1f86b0e023b21d0d92072d6636ccad89815
This commit is contained in:
committed by
copyberry
parent
4110342321
commit
b01c3986fd
@@ -175,6 +175,8 @@ impl ChatWidget {
|
||||
last_unified_wait: None,
|
||||
unified_exec_wait_streak: None,
|
||||
turn_lifecycle: TurnLifecycleState::new(prevent_idle_sleep),
|
||||
realtime_conversation: RealtimeConversationUiState::default(),
|
||||
realtime_conversation_available_for_thread: false,
|
||||
safety_buffering: SafetyBufferingState::default(),
|
||||
task_complete_pending: false,
|
||||
unified_exec_processes: Vec::new(),
|
||||
@@ -291,6 +293,9 @@ impl ChatWidget {
|
||||
widget.sync_worktrees_enabled();
|
||||
widget.sync_plugins_command_enabled();
|
||||
widget.sync_goal_command_enabled();
|
||||
widget
|
||||
.bottom_pane
|
||||
.set_voice_command_enabled(/*enabled*/ false);
|
||||
widget.sync_mentions_v2_enabled();
|
||||
widget
|
||||
.bottom_pane
|
||||
|
||||
@@ -410,6 +410,7 @@ impl ChatWidget {
|
||||
return (false, None);
|
||||
}
|
||||
self.dismiss_backend_banner_for_new_turn();
|
||||
self.note_realtime_typed_input(&submitted_message.text);
|
||||
if render_in_history {
|
||||
self.input_queue.user_turn_pending_start = true;
|
||||
}
|
||||
|
||||
@@ -26,6 +26,13 @@ impl MisalignmentReview {
|
||||
pub(super) struct MisalignmentViolation {
|
||||
turn_id: Option<String>,
|
||||
review: Option<Arc<MisalignmentReview>>,
|
||||
retired_voice: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Eq, PartialEq)]
|
||||
pub(crate) enum MisalignmentTurnSource {
|
||||
ServerNotification,
|
||||
AcknowledgedContinuation,
|
||||
}
|
||||
|
||||
const MISALIGNMENT_POLICY_TITLE: &str = "Chat stopped as a precaution";
|
||||
@@ -41,6 +48,7 @@ impl ChatWidget {
|
||||
&& !matches!(
|
||||
op,
|
||||
AppCommand::Interrupt
|
||||
| AppCommand::RealtimeConversationStop { .. }
|
||||
| AppCommand::CleanBackgroundTerminals
|
||||
| AppCommand::OverrideTurnContext { .. }
|
||||
| AppCommand::ReloadUserConfig
|
||||
@@ -80,6 +88,10 @@ impl ChatWidget {
|
||||
{
|
||||
return;
|
||||
}
|
||||
let retired_voice = self.realtime_conversation_is_running();
|
||||
if retired_voice {
|
||||
self.stop_realtime_conversation();
|
||||
}
|
||||
let review = self.thread_id.zip(turn_id.clone()).zip(details).map(
|
||||
|((thread_id, turn_id), details)| {
|
||||
Arc::new(MisalignmentReview {
|
||||
@@ -89,7 +101,11 @@ impl ChatWidget {
|
||||
})
|
||||
},
|
||||
);
|
||||
self.misalignment_policy_violation = Some(MisalignmentViolation { turn_id, review });
|
||||
self.misalignment_policy_violation = Some(MisalignmentViolation {
|
||||
turn_id,
|
||||
review,
|
||||
retired_voice,
|
||||
});
|
||||
self.input_queue.clear();
|
||||
self.finalize_turn();
|
||||
self.refresh_pending_input_preview();
|
||||
@@ -211,11 +227,19 @@ impl ChatWidget {
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn clear_misalignment_for_new_turn(&mut self, turn_id: &str) {
|
||||
pub(crate) fn clear_misalignment_for_new_turn(
|
||||
&mut self,
|
||||
turn_id: &str,
|
||||
source: MisalignmentTurnSource,
|
||||
) {
|
||||
if self
|
||||
.misalignment_policy_violation
|
||||
.as_ref()
|
||||
.is_some_and(|violation| violation.turn_id.as_ref().is_some_and(|id| id != turn_id))
|
||||
.is_some_and(|violation| {
|
||||
violation.turn_id.as_ref().is_some_and(|id| id != turn_id)
|
||||
&& (!violation.retired_voice
|
||||
|| source == MisalignmentTurnSource::AcknowledgedContinuation)
|
||||
})
|
||||
{
|
||||
self.misalignment_policy_violation = None;
|
||||
self.bottom_pane.dismiss_view_by_id(PRECAUTION_VIEW);
|
||||
|
||||
@@ -61,7 +61,10 @@ impl ChatWidget {
|
||||
}
|
||||
ServerNotification::TurnStarted(notification) => {
|
||||
if replay_kind.is_none() {
|
||||
self.clear_misalignment_for_new_turn(¬ification.turn.id);
|
||||
self.clear_misalignment_for_new_turn(
|
||||
¬ification.turn.id,
|
||||
MisalignmentTurnSource::ServerNotification,
|
||||
);
|
||||
}
|
||||
self.turn_lifecycle.last_turn_id = Some(notification.turn.id);
|
||||
self.last_non_retry_error = None;
|
||||
@@ -80,18 +83,43 @@ impl ChatWidget {
|
||||
self.handle_item_completed_notification(notification, replay_kind);
|
||||
}
|
||||
ServerNotification::AgentMessageDelta(notification) => {
|
||||
self.on_agent_message_delta(notification.delta);
|
||||
if !self.is_realtime_delegated_reasoning_turn(¬ification.turn_id)
|
||||
&& (from_replay
|
||||
|| !self.is_realtime_delegated_agent_item(
|
||||
¬ification.turn_id,
|
||||
¬ification.item_id,
|
||||
))
|
||||
{
|
||||
self.on_agent_message_delta(notification.delta);
|
||||
}
|
||||
}
|
||||
ServerNotification::PlanDelta(notification) => self.on_plan_delta(notification.delta),
|
||||
ServerNotification::ReasoningSummaryTextDelta(notification) => {
|
||||
self.on_agent_reasoning_delta(notification.delta);
|
||||
}
|
||||
ServerNotification::ReasoningTextDelta(notification) => {
|
||||
if self.config.show_raw_agent_reasoning {
|
||||
if !self.is_realtime_delegated_reasoning_item(
|
||||
¬ification.turn_id,
|
||||
¬ification.item_id,
|
||||
) {
|
||||
self.on_agent_reasoning_delta(notification.delta);
|
||||
}
|
||||
}
|
||||
ServerNotification::ReasoningSummaryPartAdded(_) => self.on_reasoning_section_break(),
|
||||
ServerNotification::ReasoningTextDelta(notification) => {
|
||||
if self.config.show_raw_agent_reasoning
|
||||
&& !self.is_realtime_delegated_reasoning_item(
|
||||
¬ification.turn_id,
|
||||
¬ification.item_id,
|
||||
)
|
||||
{
|
||||
self.on_agent_reasoning_delta(notification.delta);
|
||||
}
|
||||
}
|
||||
ServerNotification::ReasoningSummaryPartAdded(notification) => {
|
||||
if !self.is_realtime_delegated_reasoning_item(
|
||||
¬ification.turn_id,
|
||||
¬ification.item_id,
|
||||
) {
|
||||
self.on_reasoning_section_break();
|
||||
}
|
||||
}
|
||||
ServerNotification::TerminalInteraction(notification) => {
|
||||
self.on_terminal_interaction(notification.process_id, notification.stdin)
|
||||
}
|
||||
@@ -235,6 +263,36 @@ impl ChatWidget {
|
||||
self.on_shutdown_complete();
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeSdp(notification) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_conversation_sdp(notification.sdp);
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeStarted(_) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_conversation_started();
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeTranscriptDelta(notification) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_transcript_delta(notification.role, notification.delta);
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeTranscriptDone(notification) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_transcript_done(notification.role, notification.text);
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeError(notification) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_error(notification.message);
|
||||
}
|
||||
}
|
||||
ServerNotification::ThreadRealtimeClosed(notification) => {
|
||||
if !from_replay {
|
||||
self.on_realtime_conversation_closed(notification.reason);
|
||||
}
|
||||
}
|
||||
ServerNotification::ServerRequestResolved(_)
|
||||
| ServerNotification::AccountUpdated(_)
|
||||
| ServerNotification::AccountRateLimitsUpdated(_)
|
||||
@@ -264,17 +322,11 @@ impl ChatWidget {
|
||||
| ServerNotification::TurnModerationMetadata(_)
|
||||
| ServerNotification::FuzzyFileSearchSessionUpdated(_)
|
||||
| ServerNotification::FuzzyFileSearchSessionCompleted(_)
|
||||
| ServerNotification::ThreadRealtimeStarted(_)
|
||||
| ServerNotification::ThreadRealtimeItemAdded(_)
|
||||
| ServerNotification::ThreadRealtimeItemStarted(_)
|
||||
| ServerNotification::ThreadRealtimeItemTranscriptDelta(_)
|
||||
| ServerNotification::ThreadRealtimeItemCompleted(_)
|
||||
| ServerNotification::ThreadRealtimeOutputAudioDelta(_)
|
||||
| ServerNotification::ThreadRealtimeError(_)
|
||||
| ServerNotification::ThreadRealtimeClosed(_)
|
||||
| ServerNotification::ThreadRealtimeSdp(_)
|
||||
| ServerNotification::ThreadRealtimeTranscriptDelta(_)
|
||||
| ServerNotification::ThreadRealtimeTranscriptDone(_)
|
||||
| ServerNotification::WindowsWorldWritableWarning(_)
|
||||
| ServerNotification::WindowsSandboxSetupCompleted(_)
|
||||
| ServerNotification::AccountLoginCompleted(_)
|
||||
@@ -310,7 +362,12 @@ impl ChatWidget {
|
||||
text,
|
||||
phase: Some(MessagePhase::FinalAnswer) | None,
|
||||
..
|
||||
} => Some((item.clone(), id.clone(), text.clone())),
|
||||
} if !self
|
||||
.is_realtime_delegated_reasoning_turn(¬ification.turn.id)
|
||||
|| !realtime::is_private_realtime_agent_item(item) =>
|
||||
{
|
||||
Some((item.clone(), id.clone(), text.clone()))
|
||||
}
|
||||
_ => None,
|
||||
});
|
||||
if let Some((item, id, _)) = &last_agent_message
|
||||
@@ -329,6 +386,11 @@ impl ChatWidget {
|
||||
.map_or(ThreadItemRenderSource::Live, ThreadItemRenderSource::Replay),
|
||||
);
|
||||
}
|
||||
if replay_kind.is_none()
|
||||
&& let Some((item, _, _)) = &last_agent_message
|
||||
{
|
||||
self.speak_completed_realtime_delegation(¬ification.turn.id, item);
|
||||
}
|
||||
self.last_non_retry_error = None;
|
||||
let completion = self.completion_cell(¬ification.turn, replay_kind);
|
||||
self.on_task_complete(
|
||||
@@ -355,7 +417,10 @@ impl ChatWidget {
|
||||
&& error.codex_error_info
|
||||
== Some(AppServerCodexErrorInfo::MisalignmentPolicyViolation)
|
||||
{
|
||||
self.on_misalignment_error(Some(notification.turn.id), error.misalignment);
|
||||
self.on_misalignment_error(
|
||||
Some(notification.turn.id.clone()),
|
||||
error.misalignment,
|
||||
);
|
||||
self.last_non_retry_error = None;
|
||||
} else if self.last_non_retry_error.as_ref()
|
||||
== Some(&(notification.turn.id.clone(), error.message.clone()))
|
||||
@@ -373,6 +438,9 @@ impl ChatWidget {
|
||||
}
|
||||
TurnStatus::InProgress => {}
|
||||
}
|
||||
if replay_kind.is_none() {
|
||||
self.finish_realtime_turn(¬ification.turn.id);
|
||||
}
|
||||
self.thread_usage.replaying_turn_completion = was_replaying_turn_completion;
|
||||
}
|
||||
|
||||
@@ -382,6 +450,28 @@ impl ChatWidget {
|
||||
replay_kind: Option<ReplayKind>,
|
||||
) {
|
||||
match notification.item {
|
||||
ThreadItem::UserMessage { content, .. } if replay_kind.is_none() => {
|
||||
self.note_realtime_user_item_started(¬ification.turn_id, &content);
|
||||
}
|
||||
ThreadItem::UserMessage { content, .. }
|
||||
if realtime::realtime_delegation_input(&content).is_some() =>
|
||||
{
|
||||
self.remember_realtime_delegated_reasoning_turn(¬ification.turn_id);
|
||||
}
|
||||
ThreadItem::AgentMessage { id, .. } if replay_kind.is_none() => {
|
||||
self.is_realtime_delegated_agent_item(¬ification.turn_id, &id);
|
||||
}
|
||||
ThreadItem::Reasoning { id, .. }
|
||||
if replay_kind.is_none()
|
||||
&& !self.is_realtime_delegated_reasoning_turn(¬ification.turn_id) =>
|
||||
{
|
||||
// A later voice handoff can steer this turn without making an
|
||||
// already-started typed reasoning item private.
|
||||
self.realtime_conversation
|
||||
.agent_items
|
||||
.entry((notification.turn_id.clone(), id))
|
||||
.or_insert(realtime::RealtimeAgentItemOrigin::Typed);
|
||||
}
|
||||
ThreadItem::ContextCompaction { id }
|
||||
if !matches!(replay_kind, Some(ReplayKind::ResumeInitialMessages)) =>
|
||||
{
|
||||
@@ -441,6 +531,17 @@ impl ChatWidget {
|
||||
notification: ItemCompletedNotification,
|
||||
replay_kind: Option<ReplayKind>,
|
||||
) {
|
||||
if replay_kind.is_none()
|
||||
&& self.is_realtime_delegated_reasoning_turn(¬ification.turn_id)
|
||||
&& realtime::is_private_realtime_agent_item(¬ification.item)
|
||||
&& !matches!(¬ification.item, ThreadItem::AgentMessage { id, .. } | ThreadItem::Reasoning { id, .. }
|
||||
if matches!(
|
||||
self.realtime_conversation.agent_items.get(&(notification.turn_id.clone(), id.clone())),
|
||||
Some(realtime::RealtimeAgentItemOrigin::Typed)
|
||||
))
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Buffered live notifications can introduce questions; historical turn replay cannot.
|
||||
if replay_kind == Some(ReplayKind::ThreadSnapshot)
|
||||
&& let ThreadItem::AgentMessage {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,107 @@
|
||||
//! Focused visual coverage and shared setup for voice behavior tests.
|
||||
|
||||
use super::RealtimeConversationPhase;
|
||||
use crate::app_event::AppEvent;
|
||||
use crate::chatwidget::ChatWidget;
|
||||
use crate::chatwidget::tests::make_chatwidget_manual_with_sender;
|
||||
use crate::chatwidget::tests::render_bottom_popup;
|
||||
use codex_protocol::ThreadId;
|
||||
|
||||
pub(crate) fn activate_voice_for_thread(chat: &mut ChatWidget, thread_id: ThreadId) {
|
||||
chat.thread_id = Some(thread_id);
|
||||
chat.realtime_conversation.phase = RealtimeConversationPhase::Active;
|
||||
chat.realtime_conversation.thread_id = Some(thread_id);
|
||||
chat.realtime_conversation.backend_started = true;
|
||||
chat.realtime_conversation.latest_input_was_voice = true;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enabling_voice_on_an_open_thread_snapshots_the_new_thread_notice() {
|
||||
let (mut chat, _sender, mut events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
chat.set_feature_enabled(
|
||||
codex_features::Feature::RealtimeConversation,
|
||||
/*enabled*/ true,
|
||||
);
|
||||
let rendered = std::iter::from_fn(|| events.try_recv().ok())
|
||||
.filter_map(|event| match event {
|
||||
AppEvent::InsertHistoryCell(cell) => Some(
|
||||
cell.display_lines(/*width*/ 80)
|
||||
.into_iter()
|
||||
.map(|line| line.to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"),
|
||||
),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
insta::assert_snapshot!(rendered, @"• Voice conversations will be available in new threads.");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn voice_footer_renders_the_main_conversation_states() {
|
||||
let (mut chat, _sender, _events, _ops) = make_chatwidget_manual_with_sender().await;
|
||||
let mut states = Vec::new();
|
||||
|
||||
for (label, phase, muted, level, speaker_level, role, transcript) in [
|
||||
(
|
||||
"connecting",
|
||||
RealtimeConversationPhase::Starting,
|
||||
false,
|
||||
0,
|
||||
0,
|
||||
None,
|
||||
"",
|
||||
),
|
||||
(
|
||||
"listening",
|
||||
RealtimeConversationPhase::Active,
|
||||
false,
|
||||
0,
|
||||
0,
|
||||
None,
|
||||
"",
|
||||
),
|
||||
(
|
||||
"speaking",
|
||||
RealtimeConversationPhase::Active,
|
||||
false,
|
||||
4,
|
||||
5,
|
||||
None,
|
||||
"",
|
||||
),
|
||||
(
|
||||
"muted",
|
||||
RealtimeConversationPhase::Active,
|
||||
true,
|
||||
4,
|
||||
4,
|
||||
None,
|
||||
"",
|
||||
),
|
||||
(
|
||||
"transcript",
|
||||
RealtimeConversationPhase::Active,
|
||||
false,
|
||||
2,
|
||||
2,
|
||||
Some("assistant"),
|
||||
"Hello there",
|
||||
),
|
||||
] {
|
||||
chat.realtime_conversation.phase = phase;
|
||||
chat.realtime_conversation.microphone_muted = muted;
|
||||
chat.realtime_conversation.microphone_level = level;
|
||||
chat.realtime_conversation.speaker_level = speaker_level;
|
||||
chat.realtime_conversation.transcript_role = role.map(str::to_string);
|
||||
chat.realtime_conversation.transcript = transcript.to_string();
|
||||
chat.update_realtime_footer();
|
||||
states.push(format!(
|
||||
"{label}:\n{}",
|
||||
render_bottom_popup(&chat, /*width*/ 80)
|
||||
));
|
||||
}
|
||||
|
||||
insta::assert_snapshot!(states.join("\n\n"));
|
||||
}
|
||||
@@ -4,6 +4,9 @@ use super::*;
|
||||
|
||||
impl ChatWidget {
|
||||
pub(crate) fn pause_for_disconnect(&mut self) {
|
||||
// The app-server transport can fail while the separate WebRTC helper
|
||||
// still sends microphone audio. Retire local media before showing offline UI.
|
||||
let _ = self.reset_realtime_conversation();
|
||||
if let Some(questions) = &mut self.bottom_pane.questions {
|
||||
questions.delivery_enabled = false;
|
||||
}
|
||||
|
||||
@@ -158,6 +158,17 @@ impl ChatWidget {
|
||||
};
|
||||
let mut flex = FlexRenderable::new();
|
||||
flex.push(/*flex*/ 1, active_cell_renderable);
|
||||
if let Some(cell) = self.realtime_conversation.live_transcript_cell.as_ref() {
|
||||
flex.push(
|
||||
/*flex*/ 1,
|
||||
RenderableItem::Owned(Box::new(TranscriptAreaRenderable {
|
||||
child: cell.as_ref(),
|
||||
top: 1,
|
||||
right: active_cell_right_reserve,
|
||||
persistent_layout: None,
|
||||
})),
|
||||
);
|
||||
}
|
||||
if let Some(cell) = self.pending_token_activity_output() {
|
||||
flex.push(
|
||||
/*flex*/ 1,
|
||||
|
||||
@@ -46,13 +46,31 @@ impl ChatWidget {
|
||||
completed_at,
|
||||
duration_ms,
|
||||
} = turn;
|
||||
let delegated = items.iter().any(|item| {
|
||||
matches!(item, ThreadItem::UserMessage { content, .. }
|
||||
if realtime::realtime_delegation_input(content).is_some())
|
||||
});
|
||||
if matches!(status, TurnStatus::InProgress) {
|
||||
if delegated {
|
||||
self.remember_realtime_delegated_reasoning_turn(&turn_id);
|
||||
}
|
||||
self.warning_display_state.startup_complete = true;
|
||||
self.turn_lifecycle.last_turn_id = Some(turn_id.clone());
|
||||
self.last_non_retry_error = None;
|
||||
self.on_task_started();
|
||||
}
|
||||
let mut replaying_delegation = false;
|
||||
for item in items {
|
||||
if matches!(&item, ThreadItem::UserMessage { content, .. }
|
||||
if realtime::realtime_delegation_input(content).is_some())
|
||||
{
|
||||
replaying_delegation = true;
|
||||
}
|
||||
// Voice can steer a typed turn already in progress. Its earlier
|
||||
// commentary and reasoning still belong to the typed request.
|
||||
if replaying_delegation && realtime::is_private_realtime_agent_item(&item) {
|
||||
continue;
|
||||
}
|
||||
if hidden_nested_review_turn && matches!(item, ThreadItem::UserMessage { .. }) {
|
||||
continue;
|
||||
}
|
||||
@@ -122,7 +140,12 @@ impl ChatWidget {
|
||||
ThreadItem::UserMessage {
|
||||
content, client_id, ..
|
||||
} => {
|
||||
self.on_committed_user_message(&content, client_id.as_deref(), from_replay);
|
||||
self.on_committed_user_message(
|
||||
&content,
|
||||
client_id.as_deref(),
|
||||
from_replay,
|
||||
&turn_id,
|
||||
);
|
||||
}
|
||||
ThreadItem::AgentMessage {
|
||||
id,
|
||||
@@ -133,6 +156,20 @@ impl ChatWidget {
|
||||
questions,
|
||||
..
|
||||
} => {
|
||||
if self.complete_realtime_delegated_agent_item(
|
||||
&turn_id,
|
||||
&ThreadItem::AgentMessage {
|
||||
id: id.clone(),
|
||||
text: text.clone(),
|
||||
phase: phase.clone(),
|
||||
memory_citation: memory_citation.clone(),
|
||||
delivery,
|
||||
questions: questions.clone(),
|
||||
},
|
||||
from_replay,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
self.on_agent_message_item_completed(
|
||||
AgentMessageItem {
|
||||
id,
|
||||
|
||||
@@ -24,6 +24,11 @@ impl ChatWidget {
|
||||
let connector_scope_changed = previous_thread_id != Some(session.thread_id)
|
||||
|| self.config.cwd.as_path() != session.cwd.as_path();
|
||||
self.thread_id = Some(session.thread_id);
|
||||
self.realtime_conversation_available_for_thread =
|
||||
self.config.features.enabled(Feature::RealtimeConversation)
|
||||
&& codex_realtime_webrtc::RealtimeWebrtcSession::is_supported();
|
||||
self.bottom_pane
|
||||
.set_voice_command_enabled(self.realtime_conversation_available_for_thread);
|
||||
self.bottom_pane
|
||||
.set_queue_submissions(/*queue_submissions*/ false);
|
||||
if previous_thread_id != self.thread_id {
|
||||
|
||||
@@ -100,6 +100,21 @@ impl ChatWidget {
|
||||
self.update_collaboration_mode_indicator();
|
||||
}
|
||||
}
|
||||
if feature == Feature::RealtimeConversation && !enabled {
|
||||
self.realtime_conversation_available_for_thread = false;
|
||||
self.bottom_pane
|
||||
.set_voice_command_enabled(/*enabled*/ false);
|
||||
self.stop_realtime_conversation();
|
||||
}
|
||||
if feature == Feature::RealtimeConversation
|
||||
&& enabled
|
||||
&& !self.realtime_conversation_available_for_thread
|
||||
{
|
||||
self.add_info_message(
|
||||
"Voice conversations will be available in new threads.".into(),
|
||||
/*hint*/ None,
|
||||
);
|
||||
}
|
||||
if feature == Feature::MentionsV2 {
|
||||
self.sync_mentions_v2_enabled();
|
||||
}
|
||||
|
||||
@@ -333,6 +333,9 @@ impl ChatWidget {
|
||||
);
|
||||
}
|
||||
}
|
||||
SlashCommand::Voice => {
|
||||
self.toggle_realtime_conversation();
|
||||
}
|
||||
SlashCommand::Side | SlashCommand::Btw => {
|
||||
self.request_empty_side_conversation(cmd);
|
||||
}
|
||||
@@ -757,6 +760,11 @@ impl ChatWidget {
|
||||
}
|
||||
}
|
||||
}
|
||||
SlashCommand::Voice => match trimmed.to_ascii_lowercase().as_str() {
|
||||
"mute" => self.toggle_realtime_microphone(),
|
||||
"stop" => self.stop_realtime_conversation(),
|
||||
_ => self.add_error_message("Usage: /voice [mute|stop]".to_string()),
|
||||
},
|
||||
SlashCommand::Ide => {
|
||||
self.handle_ide_command_args(trimmed);
|
||||
}
|
||||
@@ -1144,6 +1152,7 @@ impl ChatWidget {
|
||||
goal_command_enabled: self.config.features.enabled(Feature::Goals),
|
||||
service_tier_commands_enabled: self.fast_mode_enabled(),
|
||||
personality_command_enabled: self.config.features.enabled(Feature::Personality),
|
||||
voice_command_enabled: self.realtime_conversation_available_for_thread,
|
||||
worktrees_enabled: self.config.features.enabled(Feature::Worktrees)
|
||||
&& self.local_worktree_operations,
|
||||
allow_elevate_sandbox,
|
||||
@@ -1183,6 +1192,7 @@ impl ChatWidget {
|
||||
| SlashCommand::Diff
|
||||
| SlashCommand::App
|
||||
| SlashCommand::Rename
|
||||
| SlashCommand::Voice
|
||||
| SlashCommand::Recap
|
||||
| SlashCommand::TestApproval => QueueDrain::Continue,
|
||||
SlashCommand::Cd => match self.thread_id {
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
---
|
||||
source: tui/src/chatwidget/realtime_tests.rs
|
||||
assertion_line: 176
|
||||
expression: "states.join(\"\\n\\n\")"
|
||||
---
|
||||
connecting:
|
||||
› Ask Codex to do anything
|
||||
|
||||
voice ◌ connecting /voice stop /voice mute
|
||||
|
||||
listening:
|
||||
› Ask Codex to do anything
|
||||
|
||||
voice ● listening /voice stop mic ░░░░░ codex ░░░░░ /voice mute
|
||||
|
||||
speaking:
|
||||
› Ask Codex to do anything
|
||||
|
||||
voice ● speaking /voice stop mic ████░ codex █████ /voice mute
|
||||
|
||||
muted:
|
||||
› Ask Codex to do anything
|
||||
|
||||
voice ◌ muted /voice stop mic off codex ████░ /voice mute
|
||||
|
||||
transcript:
|
||||
› Ask Codex to do anything
|
||||
|
||||
voice ● speaking /voice stop mic ██░░░ codex ██░░░ /voice mute
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
---
|
||||
source: tui/src/chatwidget/tests/popups_and_settings.rs
|
||||
assertion_line: 3142
|
||||
expression: popup
|
||||
---
|
||||
Experimental features
|
||||
Checked features are configured on. Some experimental features take effect
|
||||
only in new tasks or after restarting the Codex server.
|
||||
|
||||
› [ ] Network proxy Apply network proxy restrictions to
|
||||
sandboxed sessions that already have
|
||||
network access.
|
||||
[ ] Prevent sleep while running Keep your computer awake while Codex is
|
||||
running a thread.
|
||||
|
||||
Press space to select or enter to save
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
---
|
||||
source: tui/src/chatwidget/tests/status_and_layout.rs
|
||||
assertion_line: 34
|
||||
expression: rendered
|
||||
---
|
||||
" "
|
||||
"• Agent answer arriving "
|
||||
" "
|
||||
" "
|
||||
"› pick a number "
|
||||
" "
|
||||
" "
|
||||
" "
|
||||
"› Ask Codex to do anything "
|
||||
" "
|
||||
" voice ● listening /voice stop mic ░░░░░ codex ░░░░░ "
|
||||
@@ -1,8 +1,51 @@
|
||||
use super::*;
|
||||
use crate::chatwidget::realtime::tests::activate_voice_for_thread;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
const ERROR_MESSAGE: &str = "Responses API returned misalignment_policy_violation";
|
||||
|
||||
#[tokio::test]
|
||||
async fn misalignment_precaution_retires_voice_and_keeps_late_turn_blocked() {
|
||||
let (mut chat, _events, mut ops) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
let thread_id = ThreadId::new();
|
||||
chat.thread_id = Some(thread_id);
|
||||
handle_turn_started(&mut chat, "unsafe-turn");
|
||||
activate_voice_for_thread(&mut chat, thread_id);
|
||||
|
||||
handle_error(
|
||||
&mut chat,
|
||||
ERROR_MESSAGE,
|
||||
Some(CodexErrorInfo::MisalignmentPolicyViolation),
|
||||
);
|
||||
|
||||
assert!(!chat.realtime_conversation_is_running());
|
||||
assert_matches!(
|
||||
ops.try_recv(),
|
||||
Ok(Op::RealtimeConversationStop { thread_id: stopped }) if stopped == thread_id
|
||||
);
|
||||
assert!(ops.try_recv().is_err());
|
||||
assert!(chat.has_misalignment_policy_violation());
|
||||
assert!(!chat.bottom_pane.composer_input_enabled());
|
||||
assert!(chat.submit_op(Op::RealtimeConversationStop { thread_id }));
|
||||
assert_matches!(ops.try_recv(), Ok(Op::RealtimeConversationStop { .. }));
|
||||
|
||||
// A handoff already in flight may still produce a TurnStarted notification.
|
||||
// It must not dismiss the precaution or restart local capture.
|
||||
handle_turn_started(&mut chat, "late-voice-turn");
|
||||
assert!(chat.has_misalignment_policy_violation());
|
||||
assert!(!chat.bottom_pane.composer_input_enabled());
|
||||
chat.toggle_realtime_conversation();
|
||||
assert!(!chat.realtime_conversation_is_running());
|
||||
assert!(ops.try_recv().is_err());
|
||||
|
||||
chat.clear_misalignment_for_new_turn(
|
||||
"acknowledged-turn",
|
||||
MisalignmentTurnSource::AcknowledgedContinuation,
|
||||
);
|
||||
assert!(!chat.has_misalignment_policy_violation());
|
||||
assert!(chat.bottom_pane.composer_input_enabled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn misalignment_policy_failure_stops_the_thread_and_renders_once() {
|
||||
let (mut chat, mut rx, mut op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
|
||||
@@ -3175,6 +3175,35 @@ async fn experimental_popup_loading_snapshot() {
|
||||
assert!(rx.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn experimental_popup_available_snapshot() {
|
||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
chat.thread_id = Some(ThreadId::new());
|
||||
chat.open_experimental_popup();
|
||||
let AppEvent::FetchExperimentalFeatures { response_tx, .. } = rx.try_recv().unwrap() else {
|
||||
panic!("expected experimental discovery request");
|
||||
};
|
||||
let features = [
|
||||
("network_proxy", "Network proxy", "Apply network proxy restrictions to sandboxed sessions that already have network access."),
|
||||
("prevent_idle_sleep", "Prevent sleep while running", "Keep your computer awake while Codex is running a thread."),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, display_name, description)| codex_app_server_protocol::ExperimentalFeature {
|
||||
name: name.to_string(),
|
||||
stage: codex_app_server_protocol::ExperimentalFeatureStage::Beta,
|
||||
display_name: Some(display_name.to_string()),
|
||||
description: Some(description.to_string()),
|
||||
announcement: None,
|
||||
enabled: false,
|
||||
default_enabled: false,
|
||||
})
|
||||
.collect();
|
||||
response_tx.send(Ok(features)).unwrap();
|
||||
chat.pre_draw_tick();
|
||||
let popup = render_bottom_popup(&chat, /*width*/ 80);
|
||||
assert_chatwidget_snapshot!("experimental_features_available_popup", popup);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn feature_enable_prompts_snapshot() {
|
||||
let (mut chat, mut rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
|
||||
@@ -3,12 +3,37 @@ use crate::bottom_pane::goal_status_indicator_line;
|
||||
use crate::chatwidget::ThreadUsageOutcome;
|
||||
use crate::chatwidget::rate_limits::NUDGE_MODEL_SLUG;
|
||||
use crate::chatwidget::rate_limits::get_limits_duration;
|
||||
use crate::chatwidget::realtime::tests::activate_voice_for_thread;
|
||||
use codex_app_server_protocol::SpendControlLimitSnapshot;
|
||||
use codex_app_server_protocol::ThreadUsage;
|
||||
use pretty_assertions::assert_eq;
|
||||
use ratatui::Terminal;
|
||||
use ratatui::backend::TestBackend;
|
||||
use serial_test::serial;
|
||||
|
||||
#[tokio::test]
|
||||
async fn voice_live_transcript_renders_beside_the_streamed_cell() {
|
||||
let (mut chat, _rx, _ops) = make_chatwidget_manual(/*model_override*/ None).await;
|
||||
chat.config.animations = false;
|
||||
activate_voice_for_thread(&mut chat, ThreadId::new());
|
||||
chat.transcript.active_cell = Some(Box::new(history_cell::StreamingAgentTailCell::new(
|
||||
vec![Line::from("Agent answer arriving").into()],
|
||||
/*is_first_line*/ true,
|
||||
)));
|
||||
chat.on_realtime_transcript_delta("user".into(), "pick a number".into());
|
||||
|
||||
let width = 60;
|
||||
let height = chat.desired_height(width);
|
||||
let mut terminal = Terminal::new(TestBackend::new(width, height)).expect("create terminal");
|
||||
terminal
|
||||
.draw(|frame| chat.render(frame.area(), frame.buffer_mut()))
|
||||
.expect("render live voice transcript");
|
||||
let rendered = normalized_backend_snapshot(terminal.backend());
|
||||
assert!(rendered.contains("Agent answer arriving"), "{rendered}");
|
||||
assert!(rendered.contains("pick a number"), "{rendered}");
|
||||
assert_chatwidget_snapshot!("voice_live_transcript_and_stream", rendered);
|
||||
}
|
||||
|
||||
fn enable_test_ambient_pet(chat: &mut ChatWidget) {
|
||||
chat.set_pet_image_support_for_tests(crate::pets::PetImageSupport::Supported(
|
||||
crate::pets::ImageProtocol::Kitty,
|
||||
|
||||
@@ -208,7 +208,14 @@ impl ChatWidget {
|
||||
.current_goal_status
|
||||
.as_ref()
|
||||
.is_some_and(GoalStatusState::is_active);
|
||||
if !follow_up_started && !active_goal_continuing {
|
||||
if !follow_up_started
|
||||
&& !active_goal_continuing
|
||||
&& !self
|
||||
.turn_lifecycle
|
||||
.last_turn_id
|
||||
.as_deref()
|
||||
.is_some_and(|turn_id| self.should_hide_realtime_delegation(turn_id))
|
||||
{
|
||||
self.notify(Notification::AgentTurnComplete {
|
||||
response: notification_response,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user