mirror of
https://github.com/openai/codex.git
synced 2026-11-09 20:50:33 +00:00
jiayuhuang-openai
6751b54cae
## What changed - Add the optional `delegationAckFiller` field to `thread/realtime/start`. - Forward explicit `true` or `false` values to V3 Frameless Bidi session payloads as `delegation.ack_filler`; leave the field absent when unspecified so the Realtime API default is preserved. - Document that V1 and V2 ignore this setting. ## Testing - Cover both boolean values in Frameless Bidi serialization and verify the WebRTC V3 session creation payload. GitOrigin-RevId: 9fe8efd14e42738c3e86e3e9a8af81d0f4521244
273 lines
8.2 KiB
Rust
273 lines
8.2 KiB
Rust
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
|
|
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
|
|
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
|
|
use codex_protocol::protocol::ConversationTextParams;
|
|
use codex_protocol::protocol::ConversationTextRole;
|
|
pub use codex_protocol::protocol::RealtimeAudioFrame;
|
|
pub use codex_protocol::protocol::RealtimeEvent;
|
|
pub use codex_protocol::protocol::RealtimeOutputModality;
|
|
pub use codex_protocol::protocol::RealtimeTranscriptEntry;
|
|
pub use codex_protocol::protocol::RealtimeVoice;
|
|
use serde::Serialize;
|
|
use serde_json::Value;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum RealtimeEventParser {
|
|
V1,
|
|
FramelessBidi,
|
|
RealtimeV2,
|
|
}
|
|
|
|
pub type RealtimeWireAdapter = RealtimeEventParser;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum RealtimeSessionMode {
|
|
Conversational,
|
|
Transcription,
|
|
}
|
|
|
|
/// Selects the semantic stream used for Frameless Bidi context appends.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub enum RealtimeContextAppendChannel {
|
|
Speakable,
|
|
Commentary,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct RealtimeSessionConfig {
|
|
pub instructions: String,
|
|
pub initial_items: Vec<ConversationTextParams>,
|
|
pub delegation_ack_filler: Option<bool>,
|
|
pub model: Option<String>,
|
|
pub session_id: Option<String>,
|
|
pub event_parser: RealtimeEventParser,
|
|
pub session_mode: RealtimeSessionMode,
|
|
pub output_modality: RealtimeOutputModality,
|
|
pub voice: RealtimeVoice,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
#[serde(tag = "type")]
|
|
pub(super) enum RealtimeOutboundMessage {
|
|
#[serde(rename = "input_audio_buffer.append")]
|
|
InputAudioBufferAppend { audio: String },
|
|
#[serde(rename = "conversation.handoff.append")]
|
|
ConversationHandoffAppend {
|
|
handoff_id: String,
|
|
output_text: String,
|
|
},
|
|
#[serde(rename = "input_audio.append")]
|
|
InputAudioAppend { audio: String },
|
|
#[serde(rename = "delegation.context.append")]
|
|
DelegationContextAppend {
|
|
delegation_item_id: String,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
channel: Option<RealtimeContextAppendChannel>,
|
|
content: Vec<FramelessInputTextContent>,
|
|
},
|
|
#[serde(rename = "session.context.append")]
|
|
SessionContextAppend {
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
channel: Option<RealtimeContextAppendChannel>,
|
|
content: Vec<FramelessInputTextContent>,
|
|
},
|
|
#[serde(rename = "session.close")]
|
|
SessionClose,
|
|
#[serde(rename = "response.create")]
|
|
ResponseCreate,
|
|
#[serde(rename = "session.update")]
|
|
SessionUpdate { session: SessionUpdateSession },
|
|
#[serde(rename = "session.update")]
|
|
FramelessSessionUpdate { session: Value },
|
|
#[serde(rename = "conversation.item.create")]
|
|
ConversationItemCreate { item: ConversationItemPayload },
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct FramelessInputTextContent {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: FramelessContentType,
|
|
pub(super) text: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum FramelessContentType {
|
|
InputText,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionUpdateSession {
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) id: Option<String>,
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: SessionType,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) model: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) instructions: Option<String>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) output_modalities: Option<Vec<String>>,
|
|
pub(super) audio: SessionAudio,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) tools: Option<Vec<SessionFunctionTool>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) tool_choice: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum SessionType {
|
|
Quicksilver,
|
|
Realtime,
|
|
Transcription,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionAudio {
|
|
pub(super) input: SessionAudioInput,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) output: Option<SessionAudioOutput>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionAudioInput {
|
|
pub(super) format: SessionAudioFormat,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) noise_reduction: Option<SessionNoiseReduction>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) transcription: Option<SessionInputAudioTranscription>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) turn_detection: Option<SessionTurnDetection>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionInputAudioTranscription {
|
|
pub(super) model: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionAudioFormat {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: AudioFormatType,
|
|
pub(super) rate: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
pub(super) enum AudioFormatType {
|
|
#[serde(rename = "audio/pcm")]
|
|
AudioPcm,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionAudioOutput {
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub(super) format: Option<SessionAudioOutputFormat>,
|
|
pub(super) voice: RealtimeVoice,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionNoiseReduction {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: NoiseReductionType,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum NoiseReductionType {
|
|
NearField,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionTurnDetection {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: TurnDetectionType,
|
|
pub(super) interrupt_response: bool,
|
|
pub(super) create_response: bool,
|
|
pub(super) silence_duration_ms: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum TurnDetectionType {
|
|
ServerVad,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionAudioOutputFormat {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: AudioFormatType,
|
|
pub(super) rate: u32,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct ConversationMessageItem {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: ConversationItemType,
|
|
pub(super) role: ConversationTextRole,
|
|
pub(super) content: Vec<ConversationItemContent>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum ConversationItemType {
|
|
Message,
|
|
FunctionCallOutput,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
#[serde(untagged)]
|
|
pub(super) enum ConversationItemPayload {
|
|
Message(ConversationMessageItem),
|
|
FunctionCallOutput(ConversationFunctionCallOutputItem),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct ConversationFunctionCallOutputItem {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: ConversationItemType,
|
|
pub(super) call_id: String,
|
|
pub(super) output: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct ConversationItemContent {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: ConversationContentType,
|
|
pub(super) text: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum ConversationContentType {
|
|
InputText,
|
|
OutputText,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize)]
|
|
pub(super) struct SessionFunctionTool {
|
|
#[serde(rename = "type")]
|
|
pub(super) r#type: SessionToolType,
|
|
pub(super) name: String,
|
|
pub(super) description: String,
|
|
pub(super) parameters: Value,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, Serialize)]
|
|
#[serde(rename_all = "snake_case")]
|
|
pub(super) enum SessionToolType {
|
|
Function,
|
|
}
|
|
|
|
pub(super) fn parse_realtime_event(
|
|
payload: &str,
|
|
event_parser: RealtimeEventParser,
|
|
) -> Option<RealtimeEvent> {
|
|
match event_parser {
|
|
RealtimeEventParser::V1 => parse_realtime_event_v1(payload),
|
|
RealtimeEventParser::FramelessBidi => parse_frameless_bidi_event(payload),
|
|
RealtimeEventParser::RealtimeV2 => parse_realtime_event_v2(payload),
|
|
}
|
|
}
|