diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index f4c0f99ae3..95d2a62b06 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -550,9 +550,10 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { startup_context_request.body_json()["type"].as_str(), Some("session.update") ); - assert_eq!( - startup_context_request.body_json()["session"]["audio"]["output"]["voice"], - "cedar" + assert!( + startup_context_request.body_json()["session"]["audio"] + .get("output") + .is_none() ); let startup_context_instructions = startup_context_request.body_json()["session"]["instructions"] @@ -973,7 +974,7 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { Some("multipart/form-data; boundary=codex-realtime-call-boundary") ); let body = String::from_utf8(request.body).context("multipart body should be utf-8")?; - let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["audio"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}},"output":{"format":{"type":"audio/pcm","rate":24000},"voice":"marin"}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#; + let session = r#"{"tool_choice":"auto","type":"realtime","model":"gpt-realtime-1.5","instructions":"backend prompt\n\nstartup context","output_modalities":["text"],"audio":{"input":{"format":{"type":"audio/pcm","rate":24000},"noise_reduction":{"type":"near_field"},"turn_detection":{"type":"server_vad","interrupt_response":true,"create_response":true}}},"tools":[{"type":"function","name":"background_agent","description":"Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.","parameters":{"type":"object","properties":{"prompt":{"type":"string","description":"The user request to delegate to the background agent."}},"required":["prompt"],"additionalProperties":false}}]}"#; assert_eq!( body, format!( @@ -1996,6 +1997,7 @@ fn assert_v2_session_update(request: &Value) -> Result<()> { .context("v2 session.update instructions")? .contains("startup context") ); + assert_eq!(request["session"]["output_modalities"], json!(["text"])); assert_eq!( request["session"]["tools"][0]["name"].as_str(), Some("background_agent") diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs index a2681f4969..50af396006 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs @@ -1519,7 +1519,7 @@ mod tests { first_json["session"]["type"], Value::String("realtime".to_string()) ); - assert_eq!(first_json["session"]["output_modalities"], json!(["audio"])); + assert_eq!(first_json["session"]["output_modalities"], json!(["text"])); assert_eq!( first_json["session"]["audio"]["input"]["format"], json!({ @@ -1541,17 +1541,7 @@ mod tests { "create_response": true, }) ); - assert_eq!( - first_json["session"]["audio"]["output"]["format"], - json!({ - "type": "audio/pcm", - "rate": 24_000, - }) - ); - assert_eq!( - first_json["session"]["audio"]["output"]["voice"], - Value::String("cedar".to_string()) - ); + assert!(first_json["session"]["audio"].get("output").is_none()); assert_eq!( first_json["session"]["tools"][0]["type"], Value::String("function".to_string()) diff --git a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs index cfca6fce61..07a3aceee9 100644 --- a/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs +++ b/codex-rs/codex-api/src/endpoint/realtime_websocket/methods_v2.rs @@ -14,8 +14,6 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeVoice; use crate::endpoint::realtime_websocket::protocol::SessionAudio; use crate::endpoint::realtime_websocket::protocol::SessionAudioFormat; use crate::endpoint::realtime_websocket::protocol::SessionAudioInput; -use crate::endpoint::realtime_websocket::protocol::SessionAudioOutput; -use crate::endpoint::realtime_websocket::protocol::SessionAudioOutputFormat; use crate::endpoint::realtime_websocket::protocol::SessionFunctionTool; use crate::endpoint::realtime_websocket::protocol::SessionNoiseReduction; use crate::endpoint::realtime_websocket::protocol::SessionToolType; @@ -25,10 +23,10 @@ use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession; use crate::endpoint::realtime_websocket::protocol::TurnDetectionType; use serde_json::json; -const REALTIME_V2_OUTPUT_MODALITY_AUDIO: &str = "audio"; +const REALTIME_V2_OUTPUT_MODALITY_TEXT: &str = "text"; const REALTIME_V2_TOOL_CHOICE: &str = "auto"; const REALTIME_V2_BACKGROUND_AGENT_TOOL_NAME: &str = "background_agent"; -const REALTIME_V2_BACKGROUND_AGENT_TOOL_DESCRIPTION: &str = "Send a user request to the background agent. Use this as the default action. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later."; +const REALTIME_V2_BACKGROUND_AGENT_TOOL_DESCRIPTION: &str = "Send a user request to the background agent. Use this as the default action. Do not rephrase the user's ask or rewrite it in your own words; pass along the user's own words. If the background agent is idle, this starts a new task and returns the final result to the user. If the background agent is already working on a task, this sends the request as guidance to steer that previous task. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later."; pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage { RealtimeOutboundMessage::ConversationItemCreate { @@ -59,7 +57,7 @@ pub(super) fn conversation_handoff_append_message( pub(super) fn session_update_session( instructions: String, session_mode: RealtimeSessionMode, - voice: RealtimeVoice, + _voice: RealtimeVoice, ) -> SessionUpdateSession { match session_mode { RealtimeSessionMode::Conversational => SessionUpdateSession { @@ -67,7 +65,7 @@ pub(super) fn session_update_session( r#type: SessionType::Realtime, model: None, instructions: Some(instructions), - output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_AUDIO.to_string()]), + output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_TEXT.to_string()]), audio: SessionAudio { input: SessionAudioInput { format: SessionAudioFormat { @@ -83,13 +81,7 @@ pub(super) fn session_update_session( create_response: true, }), }, - output: Some(SessionAudioOutput { - format: Some(SessionAudioOutputFormat { - r#type: AudioFormatType::AudioPcm, - rate: REALTIME_AUDIO_SAMPLE_RATE, - }), - voice, - }), + output: None, }, tools: Some(vec![SessionFunctionTool { r#type: SessionToolType::Function, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index efb4c04194..b1b1bfb9cf 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -193,6 +193,10 @@ use codex_protocol::protocol::McpToolCallEndEvent; use codex_protocol::protocol::Op; use codex_protocol::protocol::PatchApplyBeginEvent; use codex_protocol::protocol::RateLimitSnapshot; +use codex_protocol::protocol::RealtimeConversationClosedEvent; +use codex_protocol::protocol::RealtimeConversationRealtimeEvent; +use codex_protocol::protocol::RealtimeConversationStartedEvent; +use codex_protocol::protocol::RealtimeEvent; use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::ReviewTarget; use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata; @@ -337,6 +341,7 @@ use crate::history_cell::HistoryCell; use crate::history_cell::HookCell; use crate::history_cell::McpToolCallCell; use crate::history_cell::PlainHistoryCell; +use crate::history_cell::RealtimeTranscriptRole; use crate::history_cell::WebSearchCell; use crate::key_hint; use crate::key_hint::KeyBinding; @@ -6175,54 +6180,45 @@ impl ChatWidget { } ServerNotification::ThreadRealtimeStarted(notification) => { if !from_replay { - self.on_realtime_conversation_started( - codex_protocol::protocol::RealtimeConversationStartedEvent { - session_id: notification.session_id, - version: notification.version, - }, - ); + self.on_realtime_conversation_started(RealtimeConversationStartedEvent { + session_id: notification.session_id, + version: notification.version, + }); } } ServerNotification::ThreadRealtimeItemAdded(notification) => { if !from_replay { - self.on_realtime_conversation_realtime( - codex_protocol::protocol::RealtimeConversationRealtimeEvent { - payload: codex_protocol::protocol::RealtimeEvent::ConversationItemAdded( - notification.item, - ), - }, - ); + self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::ConversationItemAdded(notification.item), + }); + } + } + ServerNotification::ThreadRealtimeTranscriptUpdated(notification) => { + if !from_replay + && let Some(role) = RealtimeTranscriptRole::from_name(¬ification.role) + { + self.on_realtime_transcript_delta(role, notification.text); } } ServerNotification::ThreadRealtimeOutputAudioDelta(notification) => { if !from_replay { - self.on_realtime_conversation_realtime( - codex_protocol::protocol::RealtimeConversationRealtimeEvent { - payload: codex_protocol::protocol::RealtimeEvent::AudioOut( - notification.audio.into(), - ), - }, - ); + self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::AudioOut(notification.audio.into()), + }); } } ServerNotification::ThreadRealtimeError(notification) => { if !from_replay { - self.on_realtime_conversation_realtime( - codex_protocol::protocol::RealtimeConversationRealtimeEvent { - payload: codex_protocol::protocol::RealtimeEvent::Error( - notification.message, - ), - }, - ); + self.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::Error(notification.message), + }); } } ServerNotification::ThreadRealtimeClosed(notification) => { if !from_replay { - self.on_realtime_conversation_closed( - codex_protocol::protocol::RealtimeConversationClosedEvent { - reason: notification.reason, - }, - ); + self.on_realtime_conversation_closed(RealtimeConversationClosedEvent { + reason: notification.reason, + }); } } ServerNotification::ThreadRealtimeSdp(notification) => { @@ -6245,7 +6241,6 @@ impl ChatWidget { | ServerNotification::FsChanged(_) | ServerNotification::FuzzyFileSearchSessionUpdated(_) | ServerNotification::FuzzyFileSearchSessionCompleted(_) - | ServerNotification::ThreadRealtimeTranscriptUpdated(_) | ServerNotification::WindowsWorldWritableWarning(_) | ServerNotification::WindowsSandboxSetupCompleted(_) | ServerNotification::AccountLoginCompleted(_) => {} diff --git a/codex-rs/tui/src/chatwidget/realtime.rs b/codex-rs/tui/src/chatwidget/realtime.rs index 03a59c224c..c492d4d3a9 100644 --- a/codex-rs/tui/src/chatwidget/realtime.rs +++ b/codex-rs/tui/src/chatwidget/realtime.rs @@ -1,4 +1,6 @@ use super::*; +use crate::history_cell::RealtimeTranscriptCell; +use crate::history_cell::RealtimeTranscriptRole; use codex_config::config_toml::RealtimeTransport; use codex_protocol::protocol::ConversationStartParams; use codex_protocol::protocol::ConversationStartTransport; @@ -315,8 +317,6 @@ impl ChatWidget { RealtimeEvent::AudioOut(_) | RealtimeEvent::InputAudioSpeechStarted(_) | RealtimeEvent::ResponseCreated(_) - | RealtimeEvent::ResponseCancelled(_) - | RealtimeEvent::ResponseDone(_) ) { return; @@ -326,12 +326,19 @@ impl ChatWidget { self.realtime_conversation.session_id = Some(session_id); } RealtimeEvent::InputAudioSpeechStarted(_) => self.interrupt_realtime_audio_playback(), - RealtimeEvent::InputTranscriptDelta(_) => {} - RealtimeEvent::OutputTranscriptDelta(_) => {} + RealtimeEvent::InputTranscriptDelta(delta) => { + self.on_realtime_transcript_delta(RealtimeTranscriptRole::User, delta.delta); + } + RealtimeEvent::OutputTranscriptDelta(delta) => { + self.on_realtime_transcript_delta(RealtimeTranscriptRole::Assistant, delta.delta); + } RealtimeEvent::AudioOut(frame) => self.enqueue_realtime_audio_out(&frame), RealtimeEvent::ResponseCreated(_) => {} - RealtimeEvent::ResponseCancelled(_) => self.interrupt_realtime_audio_playback(), - RealtimeEvent::ResponseDone(_) => {} + RealtimeEvent::ResponseCancelled(_) => { + self.flush_active_realtime_transcript(); + self.interrupt_realtime_audio_playback(); + } + RealtimeEvent::ResponseDone(_) => self.flush_active_realtime_transcript(), RealtimeEvent::ConversationItemAdded(_item) => {} RealtimeEvent::ConversationItemDone { .. } => {} RealtimeEvent::HandoffRequested(_) => {} @@ -341,6 +348,46 @@ impl ChatWidget { } } + pub(super) fn on_realtime_transcript_delta( + &mut self, + role: RealtimeTranscriptRole, + delta: String, + ) { + if delta.is_empty() { + return; + } + + self.flush_unified_exec_wait_streak(); + + if let Some(cell) = self + .active_cell + .as_mut() + .and_then(|cell| cell.as_any_mut().downcast_mut::()) + && cell.role == role + { + cell.append(&delta); + self.bump_active_cell_revision(); + self.request_redraw(); + return; + } + + self.flush_active_cell(); + self.active_cell = Some(Box::new(RealtimeTranscriptCell::new(role, delta))); + self.bump_active_cell_revision(); + self.request_redraw(); + } + + fn flush_active_realtime_transcript(&mut self) { + if self + .active_cell + .as_ref() + .is_some_and(|cell| cell.as_any().is::()) + { + self.flush_active_cell(); + self.request_redraw(); + } + } + pub(super) fn on_realtime_conversation_closed(&mut self, ev: RealtimeConversationClosedEvent) { if self.realtime_conversation_uses_webrtc() && self.realtime_conversation.is_live() @@ -349,6 +396,7 @@ impl ChatWidget { return; } + self.flush_active_realtime_transcript(); let requested = self.realtime_conversation.requested_close; let reason = ev.reason; self.reset_realtime_conversation_state(); diff --git a/codex-rs/tui/src/chatwidget/tests.rs b/codex-rs/tui/src/chatwidget/tests.rs index e92a809e80..8ea090923f 100644 --- a/codex-rs/tui/src/chatwidget/tests.rs +++ b/codex-rs/tui/src/chatwidget/tests.rs @@ -172,6 +172,7 @@ pub(super) use codex_protocol::protocol::ReadOnlyAccess; pub(super) use codex_protocol::protocol::RealtimeConversationClosedEvent; pub(super) use codex_protocol::protocol::RealtimeConversationRealtimeEvent; pub(super) use codex_protocol::protocol::RealtimeEvent; +pub(super) use codex_protocol::protocol::RealtimeTranscriptDelta; pub(super) use codex_protocol::protocol::ReviewRequest; pub(super) use codex_protocol::protocol::ReviewTarget; pub(super) use codex_protocol::protocol::SessionConfiguredEvent; diff --git a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs index e2635d322d..936990f481 100644 --- a/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs +++ b/codex-rs/tui/src/chatwidget/tests/popups_and_settings.rs @@ -22,6 +22,31 @@ async fn realtime_error_closes_without_followup_closed_info() { insta::assert_snapshot!(rendered.join("\n\n"), @"■ Realtime voice error: boom"); } +#[tokio::test] +async fn realtime_output_text_delta_renders_transcript_cell() { + let (mut chat, _rx, _op_rx) = make_chatwidget_manual(/*model_override*/ None).await; + chat.realtime_conversation.phase = RealtimeConversationPhase::Active; + + chat.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "hello ".to_string(), + }), + }); + chat.on_realtime_conversation_realtime(RealtimeConversationRealtimeEvent { + payload: RealtimeEvent::OutputTranscriptDelta(RealtimeTranscriptDelta { + delta: "there".to_string(), + }), + }); + + let lines = chat + .active_cell_transcript_lines(/*width*/ 80) + .expect("realtime transcript cell"); + insta::assert_snapshot!( + lines_to_single_string(&lines).trim_end(), + @"assistant: hello there" + ); +} + #[cfg(not(target_os = "linux"))] #[tokio::test] async fn deleted_realtime_meter_uses_shared_stop_path() { diff --git a/codex-rs/tui/src/history_cell.rs b/codex-rs/tui/src/history_cell.rs index b2191d3b26..aa6fba59bc 100644 --- a/codex-rs/tui/src/history_cell.rs +++ b/codex-rs/tui/src/history_cell.rs @@ -489,6 +489,98 @@ impl HistoryCell for AgentMessageCell { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RealtimeTranscriptRole { + User, + Assistant, +} + +impl RealtimeTranscriptRole { + pub(crate) fn from_name(role: &str) -> Option { + match role { + "user" => Some(Self::User), + "assistant" => Some(Self::Assistant), + _ => None, + } + } +} + +#[derive(Debug)] +pub(crate) struct RealtimeTranscriptCell { + pub(crate) role: RealtimeTranscriptRole, + text: String, +} + +impl RealtimeTranscriptCell { + pub(crate) fn new(role: RealtimeTranscriptRole, text: String) -> Self { + Self { role, text } + } + + pub(crate) fn append(&mut self, delta: &str) { + self.text.push_str(delta); + } +} + +impl HistoryCell for RealtimeTranscriptCell { + fn display_lines(&self, width: u16) -> Vec> { + let role_name = match self.role { + RealtimeTranscriptRole::User => "user", + RealtimeTranscriptRole::Assistant => "assistant", + }; + let style = Style::default().dim().italic(); + let text = match self.role { + RealtimeTranscriptRole::User => self.text.clone(), + RealtimeTranscriptRole::Assistant => clean_assistant_realtime_transcript(&self.text), + }; + let line = Line::from(text).style(style); + adaptive_wrap_lines( + &[line], + RtOptions::new(width as usize) + .initial_indent(vec![format!("{role_name}: ").dim().italic()].into()) + .subsequent_indent(" ".dim().italic().into()), + ) + } +} + +fn clean_assistant_realtime_transcript(text: &str) -> String { + let text = strip_realtime_code_fence(text); + serde_json::from_str::(text) + .ok() + .and_then(|value| { + value + .as_object() + .and_then(|object| object.get("response")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + }) + .unwrap_or_else(|| text.to_string()) +} + +fn strip_realtime_code_fence(text: &str) -> &str { + let text = text.trim(); + let Some(mut rest) = text.strip_prefix("```") else { + return text; + }; + + let has_language = rest.chars().next().is_some_and(|ch| !ch.is_whitespace()); + rest = rest.trim_start(); + if has_language && let Some((first_line, remaining)) = rest.split_once('\n') { + let first_line = first_line.trim(); + if !first_line.is_empty() + && first_line + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_') + { + rest = remaining; + } + } + + rest.trim() + .strip_suffix("```") + .map(str::trim) + .unwrap_or_else(|| rest.trim()) +} + #[derive(Debug)] pub(crate) struct PlainHistoryCell { lines: Vec>, @@ -2934,6 +3026,32 @@ mod tests { render_lines(&cell.transcript_lines(u16::MAX)) } + #[test] + fn realtime_transcript_cell_strips_assistant_code_fence() { + let cell = RealtimeTranscriptCell::new( + RealtimeTranscriptRole::Assistant, + "``` \nHey!".to_string(), + ); + + assert_eq!( + render_lines(&cell.display_lines(/*width*/ 80)), + vec!["assistant: Hey!"] + ); + } + + #[test] + fn realtime_transcript_cell_unwraps_assistant_response_json() { + let cell = RealtimeTranscriptCell::new( + RealtimeTranscriptRole::Assistant, + r#"{"response":"I'm doing well."}"#.to_string(), + ); + + assert_eq!( + render_lines(&cell.display_lines(/*width*/ 80)), + vec!["assistant: I'm doing well."] + ); + } + fn image_block(data: &str) -> serde_json::Value { serde_json::to_value(Content::image(data.to_string(), "image/png")) .expect("image content should serialize")