openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-08-23 13:09:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Route realtime V3 handoffs by response channel (#33903)

Browse Source 
## What changed

- Add `codexResponseHandoffMode` to `thread/realtime/start` for V3 sessions, with `thinking` as the default and `commentary` and `bemTags` routing modes.
- Map BEM `analysis` and `commentary` output to the commentary channel and `final` output to the speakable channel, while preserving the original BEM envelope. Unrecognized output falls back to speakable.
- Mark explicit speech appends as speakable and stop adding the legacy `"Agent Final Message"` prefix to V3 handoffs. V1 and V2 continue to ignore the new setting.

## Testing

- Cover channel selection for every routing mode, streamed BEM header parsing, fallback behavior, explicit speech, and V1 compatibility.

GitOrigin-RevId: 1974578257488f2914b252c9c1990ba38831c96b
This commit is contained in:
jiayuhuang-openai
2026-07-17 21:03:05 +00:00
committed by copyberry
parent 161748a68e
commit 025db22058
26 changed files with 851 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
codex-rs/codex-api/src/endpoint/mod.rs
View File

@@ -15,6 +15,7 @@ pub use memories::MemoriesClient;
pub use models::ModelsClient;
pub use realtime_call::RealtimeCallClient;
pub use realtime_call::RealtimeCallResponse;
pub use realtime_websocket::RealtimeContextAppendChannel;
pub use realtime_websocket::RealtimeEventParser;
pub use realtime_websocket::RealtimeOutputModality;
pub use realtime_websocket::RealtimeSessionConfig;

22
codex-rs/codex-api/src/endpoint/realtime_websocket/methods.rs
View File

@@ -9,6 +9,7 @@ use crate::endpoint::realtime_websocket::methods_frameless_bidi::context_append_
use crate::endpoint::realtime_websocket::methods_frameless_bidi::delegation_context_append_message as frameless_delegation_context_append_message;
use crate::endpoint::realtime_websocket::methods_frameless_bidi::session_context_append_message as frameless_session_context_append_message;
use crate::endpoint::realtime_websocket::protocol::RealtimeAudioFrame;
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
use crate::endpoint::realtime_websocket::protocol::RealtimeEvent;
use crate::endpoint::realtime_websocket::protocol::RealtimeEventParser;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
@@ -211,6 +212,7 @@ pub struct RealtimeWebsocketWriter {
stream: Arc<WsStream>,
is_closed: Arc<AtomicBool>,
event_parser: RealtimeEventParser,
context_append_channel: Option<RealtimeContextAppendChannel>,
}
#[derive(Clone)]
@@ -281,6 +283,7 @@ impl RealtimeWebsocketConnection {
stream: Arc::clone(&stream),
is_closed: Arc::clone(&is_closed),
event_parser,
context_append_channel: None,
},
events: RealtimeWebsocketEvents {
rx_message,
@@ -294,6 +297,11 @@ impl RealtimeWebsocketConnection {
}
impl RealtimeWebsocketWriter {
pub fn with_context_append_channel(mut self, channel: RealtimeContextAppendChannel) -> Self {
self.context_append_channel = Some(channel);
self
}
pub async fn send_audio_frame(&self, frame: RealtimeAudioFrame) -> Result<(), ApiError> {
let message = match self.event_parser {
RealtimeEventParser::V1 | RealtimeEventParser::RealtimeV2 => {
@@ -315,6 +323,7 @@ impl RealtimeWebsocketWriter {
self.event_parser,
text,
role,
self.context_append_channel,
))
.await
}
@@ -328,6 +337,7 @@ impl RealtimeWebsocketWriter {
self.event_parser,
handoff_id,
output_text,
self.context_append_channel,
))
.await
}
@@ -341,6 +351,7 @@ impl RealtimeWebsocketWriter {
self.event_parser,
handoff_id,
output_text,
self.context_append_channel,
))
.await
}
@@ -354,6 +365,7 @@ impl RealtimeWebsocketWriter {
self.event_parser,
call_id,
output_text,
self.context_append_channel,
))
.await
}
@@ -413,6 +425,7 @@ impl RealtimeWebsocketWriter {
match message {
RealtimeOutboundMessage::DelegationContextAppend {
delegation_item_id,
channel,
content,
} => {
if let Some(content) = content.first() {
@@ -420,17 +433,20 @@ impl RealtimeWebsocketWriter {
self.send_json_frame(&frameless_delegation_context_append_message(
delegation_item_id.clone(),
chunk,
*channel,
))
.await?;
}
return Ok(());
}
}
RealtimeOutboundMessage::SessionContextAppend { content } => {
RealtimeOutboundMessage::SessionContextAppend { channel, content } => {
if let Some(content) = content.first() {
for chunk in context_append_chunks(&content.text) {
self.send_json_frame(&frameless_session_context_append_message(chunk))
.await?;
self.send_json_frame(&frameless_session_context_append_message(
chunk, *channel,
))
.await?;
}
return Ok(());
}

24
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common.rs
View File

@@ -10,6 +10,7 @@ use crate::endpoint::realtime_websocket::methods_v2::conversation_function_call_
use crate::endpoint::realtime_websocket::methods_v2::conversation_item_create_message as v2_conversation_item_create_message;
use crate::endpoint::realtime_websocket::methods_v2::session_update_session as v2_session_update_session;
use crate::endpoint::realtime_websocket::methods_v2::websocket_intent as v2_websocket_intent;
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutputModality;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
@@ -40,10 +41,13 @@ pub(super) fn conversation_item_create_message(
wire_adapter: RealtimeWireAdapter,
text: String,
role: ConversationTextRole,
context_append_channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_item_create_message(text, role),
RealtimeWireAdapter::FramelessBidi => frameless_session_context_append_message(text),
RealtimeWireAdapter::FramelessBidi => {
frameless_session_context_append_message(text, context_append_channel)
}
RealtimeWireAdapter::RealtimeV2 => v2_conversation_item_create_message(text, role),
}
}
@@ -52,12 +56,15 @@ pub(super) fn conversation_handoff_append_message(
wire_adapter: RealtimeWireAdapter,
handoff_id: String,
output_text: String,
context_append_channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
RealtimeWireAdapter::FramelessBidi => {
frameless_delegation_context_append_message(handoff_id, output_text)
}
RealtimeWireAdapter::FramelessBidi => frameless_delegation_context_append_message(
handoff_id,
output_text,
context_append_channel,
),
RealtimeWireAdapter::RealtimeV2 => {
unreachable!("realtime v2 does not send conversation handoff output")
}
@@ -68,10 +75,13 @@ pub(super) fn standalone_handoff_message(
wire_adapter: RealtimeWireAdapter,
handoff_id: String,
output_text: String,
context_append_channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
RealtimeWireAdapter::FramelessBidi => frameless_session_context_append_message(output_text),
RealtimeWireAdapter::FramelessBidi => {
frameless_session_context_append_message(output_text, context_append_channel)
}
RealtimeWireAdapter::RealtimeV2 => {
unreachable!("realtime v2 does not send standalone handoff output")
}
@@ -82,6 +92,7 @@ pub(super) fn conversation_function_call_output_message(
wire_adapter: RealtimeWireAdapter,
call_id: String,
output_text: String,
context_append_channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
match wire_adapter {
RealtimeWireAdapter::V1 => v1_conversation_handoff_append_message(
@@ -90,7 +101,8 @@ pub(super) fn conversation_function_call_output_message(
),
RealtimeWireAdapter::FramelessBidi => frameless_delegation_context_append_message(
call_id,
format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}"),
output_text,
context_append_channel,
),
RealtimeWireAdapter::RealtimeV2 => {
v2_conversation_function_call_output_message(call_id, output_text)

32
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_common_tests.rs
View File

@@ -1,6 +1,7 @@
use super::conversation_function_call_output_message;
use super::conversation_handoff_append_message;
use super::standalone_handoff_message;
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
use crate::endpoint::realtime_websocket::protocol::RealtimeWireAdapter;
use pretty_assertions::assert_eq;
use serde_json::Value;
@@ -8,16 +9,18 @@ use serde_json::json;
use serde_json::to_value;
#[test]
fn identical_handoff_output_encodes_for_each_bidi_wire_protocol() {
fn context_append_channel_only_encodes_for_frameless_handoff_output() {
let legacy = conversation_handoff_append_message(
RealtimeWireAdapter::V1,
"handoff-123".to_string(),
"The result".to_string(),
Some(RealtimeContextAppendChannel::Commentary),
);
let frameless = conversation_handoff_append_message(
RealtimeWireAdapter::FramelessBidi,
"handoff-123".to_string(),
"The result".to_string(),
Some(RealtimeContextAppendChannel::Commentary),
);
assert_eq!(
@@ -33,6 +36,7 @@ fn identical_handoff_output_encodes_for_each_bidi_wire_protocol() {
json!({
"type": "delegation.context.append",
"delegation_item_id": "handoff-123",
"channel": "commentary",
"content": [{"type": "input_text", "text": "The result"}],
})
);
@@ -44,11 +48,13 @@ fn standalone_handoff_uses_session_context_for_frameless() {
RealtimeWireAdapter::V1,
"codex".to_string(),
"Speak this".to_string(),
Some(RealtimeContextAppendChannel::Speakable),
);
let frameless = standalone_handoff_message(
RealtimeWireAdapter::FramelessBidi,
"codex".to_string(),
"Speak this".to_string(),
Some(RealtimeContextAppendChannel::Speakable),
);
assert_eq!(
@@ -63,19 +69,20 @@ fn standalone_handoff_uses_session_context_for_frameless() {
to_value(frameless).expect("frameless standalone handoff should serialize"),
json!({
"type": "session.context.append",
"channel": "speakable",
"content": [{"type": "input_text", "text": "Speak this"}],
})
);
}
#[test]
fn completed_handoff_preserves_legacy_payload_text_in_frameless() {
let expected_text = Value::String("\"Agent Final Message\":\n\nDone".to_string());
fn completed_handoff_only_prefixes_v1_payload_text() {
for wire_adapter in [RealtimeWireAdapter::V1, RealtimeWireAdapter::FramelessBidi] {
let encoded = to_value(conversation_function_call_output_message(
wire_adapter,
"handoff-123".to_string(),
"Done".to_string(),
Some(RealtimeContextAppendChannel::Speakable),
))
.expect("handoff output should serialize");
let text = match wire_adapter {
@@ -83,6 +90,23 @@ fn completed_handoff_preserves_legacy_payload_text_in_frameless() {
RealtimeWireAdapter::FramelessBidi => &encoded["content"][0]["text"],
RealtimeWireAdapter::RealtimeV2 => unreachable!(),
};
assert_eq!(text, &expected_text);
assert_eq!(
text,
&Value::String(match wire_adapter {
RealtimeWireAdapter::V1 => "\"Agent Final Message\":\n\nDone".to_string(),
RealtimeWireAdapter::FramelessBidi => "Done".to_string(),
RealtimeWireAdapter::RealtimeV2 => unreachable!(),
})
);
assert_eq!(
encoded.get("channel").cloned(),
match wire_adapter {
RealtimeWireAdapter::V1 => None,
RealtimeWireAdapter::FramelessBidi => {
Some(Value::String("speakable".to_string()))
}
RealtimeWireAdapter::RealtimeV2 => unreachable!(),
}
);
}
}

9
codex-rs/codex-api/src/endpoint/realtime_websocket/methods_frameless_bidi.rs
View File

@@ -1,5 +1,6 @@
use crate::endpoint::realtime_websocket::protocol::FramelessContentType;
use crate::endpoint::realtime_websocket::protocol::FramelessInputTextContent;
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use serde_json::Value;
@@ -10,15 +11,21 @@ const CONTEXT_APPEND_MAX_BYTES: usize = 500;
pub(super) fn delegation_context_append_message(
delegation_item_id: String,
text: String,
channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::DelegationContextAppend {
delegation_item_id,
channel,
content: input_text_content(text),
}
}
pub(super) fn session_context_append_message(text: String) -> RealtimeOutboundMessage {
pub(super) fn session_context_append_message(
text: String,
channel: Option<RealtimeContextAppendChannel>,
) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::SessionContextAppend {
channel,
content: input_text_content(text),
}
}

1
codex-rs/codex-api/src/endpoint/realtime_websocket/mod.rs
View File

@@ -14,6 +14,7 @@ pub use methods::RealtimeWebsocketConnection;
pub use methods::RealtimeWebsocketEvents;
pub use methods::RealtimeWebsocketWriter;
pub use methods_common::session_update_session_json;
pub use protocol::RealtimeContextAppendChannel;
pub use protocol::RealtimeEventParser;
pub use protocol::RealtimeOutputModality;
pub use protocol::RealtimeSessionConfig;

12
codex-rs/codex-api/src/endpoint/realtime_websocket/protocol.rs
View File

@@ -25,6 +25,14 @@ pub enum RealtimeSessionMode {
Transcription,
}
/// Selects the semantic stream used for Frameless Bidi context appends.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum RealtimeContextAppendChannel {
Speakable,
Commentary,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RealtimeSessionConfig {
pub instructions: String,
@@ -51,10 +59,14 @@ pub(super) enum RealtimeOutboundMessage {
#[serde(rename = "delegation.context.append")]
DelegationContextAppend {
delegation_item_id: String,
#[serde(skip_serializing_if = "Option::is_none")]
channel: Option<RealtimeContextAppendChannel>,
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.context.append")]
SessionContextAppend {
#[serde(skip_serializing_if = "Option::is_none")]
channel: Option<RealtimeContextAppendChannel>,
content: Vec<FramelessInputTextContent>,
},
#[serde(rename = "session.close")]

1
codex-rs/codex-api/src/lib.rs
View File

@@ -52,6 +52,7 @@ pub use crate::endpoint::MemoriesClient;
pub use crate::endpoint::ModelsClient;
pub use crate::endpoint::RealtimeCallClient;
pub use crate::endpoint::RealtimeCallResponse;
pub use crate::endpoint::RealtimeContextAppendChannel;
pub use crate::endpoint::RealtimeEventParser;
pub use crate::endpoint::RealtimeOutputModality;
pub use crate::endpoint::RealtimeSessionConfig;