mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Merge branch 'main' into rhan/surface-updates
This commit is contained in:
@@ -5,6 +5,7 @@ use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::openai_models::ReasoningEffort as ReasoningEffortConfig;
|
||||
use codex_protocol::protocol::RateLimitSnapshot;
|
||||
use codex_protocol::protocol::TokenUsage;
|
||||
use codex_protocol::protocol::W3cTraceContext;
|
||||
use futures::Stream;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
@@ -15,6 +16,9 @@ use std::task::Context;
|
||||
use std::task::Poll;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub const WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY: &str = "ws_request_header_traceparent";
|
||||
pub const WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY: &str = "ws_request_header_tracestate";
|
||||
|
||||
/// Canonical input payload for the compaction endpoint.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct CompactionInput<'a> {
|
||||
@@ -215,6 +219,28 @@ pub struct ResponseCreateWsRequest {
|
||||
pub client_metadata: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
pub fn response_create_client_metadata(
|
||||
client_metadata: Option<HashMap<String, String>>,
|
||||
trace: Option<&W3cTraceContext>,
|
||||
) -> Option<HashMap<String, String>> {
|
||||
let mut client_metadata = client_metadata.unwrap_or_default();
|
||||
|
||||
if let Some(traceparent) = trace.and_then(|trace| trace.traceparent.as_deref()) {
|
||||
client_metadata.insert(
|
||||
WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY.to_string(),
|
||||
traceparent.to_string(),
|
||||
);
|
||||
}
|
||||
if let Some(tracestate) = trace.and_then(|trace| trace.tracestate.as_deref()) {
|
||||
client_metadata.insert(
|
||||
WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY.to_string(),
|
||||
tracestate.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
(!client_metadata.is_empty()).then_some(client_metadata)
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
|
||||
@@ -272,12 +272,12 @@ impl RealtimeWebsocketConnection {
|
||||
|
||||
impl RealtimeWebsocketWriter {
|
||||
pub async fn send_audio_frame(&self, frame: RealtimeAudioFrame) -> Result<(), ApiError> {
|
||||
self.send_json(RealtimeOutboundMessage::InputAudioBufferAppend { audio: frame.data })
|
||||
self.send_json(&RealtimeOutboundMessage::InputAudioBufferAppend { audio: frame.data })
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn send_conversation_item_create(&self, text: String) -> Result<(), ApiError> {
|
||||
self.send_json(conversation_item_create_message(self.event_parser, text))
|
||||
self.send_json(&conversation_item_create_message(self.event_parser, text))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -286,7 +286,7 @@ impl RealtimeWebsocketWriter {
|
||||
handoff_id: String,
|
||||
output_text: String,
|
||||
) -> Result<(), ApiError> {
|
||||
self.send_json(conversation_handoff_append_message(
|
||||
self.send_json(&conversation_handoff_append_message(
|
||||
self.event_parser,
|
||||
handoff_id,
|
||||
output_text,
|
||||
@@ -294,6 +294,11 @@ impl RealtimeWebsocketWriter {
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn send_response_create(&self) -> Result<(), ApiError> {
|
||||
self.send_json(&RealtimeOutboundMessage::ResponseCreate)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn send_session_update(
|
||||
&self,
|
||||
instructions: String,
|
||||
@@ -301,7 +306,7 @@ impl RealtimeWebsocketWriter {
|
||||
) -> Result<(), ApiError> {
|
||||
let session_mode = normalized_session_mode(self.event_parser, session_mode);
|
||||
let session = session_update_session(self.event_parser, instructions, session_mode);
|
||||
self.send_json(RealtimeOutboundMessage::SessionUpdate { session })
|
||||
self.send_json(&RealtimeOutboundMessage::SessionUpdate { session })
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -319,11 +324,14 @@ impl RealtimeWebsocketWriter {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn send_json(&self, message: RealtimeOutboundMessage) -> Result<(), ApiError> {
|
||||
let payload = serde_json::to_string(&message)
|
||||
async fn send_json(&self, message: &RealtimeOutboundMessage) -> Result<(), ApiError> {
|
||||
let payload = serde_json::to_string(message)
|
||||
.map_err(|err| ApiError::Stream(format!("failed to encode realtime request: {err}")))?;
|
||||
debug!(?message, "realtime websocket request");
|
||||
self.send_payload(payload).await
|
||||
}
|
||||
|
||||
pub async fn send_payload(&self, payload: String) -> Result<(), ApiError> {
|
||||
if self.is_closed.load(Ordering::SeqCst) {
|
||||
return Err(ApiError::Stream(
|
||||
"realtime websocket connection is closed".to_string(),
|
||||
@@ -392,6 +400,7 @@ impl RealtimeWebsocketEvents {
|
||||
async fn update_active_transcript(&self, event: &mut RealtimeEvent) {
|
||||
let mut active_transcript = self.active_transcript.lock().await;
|
||||
match event {
|
||||
RealtimeEvent::InputAudioSpeechStarted(_) => {}
|
||||
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta { delta }) => {
|
||||
append_transcript_delta(&mut active_transcript.entries, "user", delta);
|
||||
}
|
||||
@@ -399,10 +408,13 @@ impl RealtimeWebsocketEvents {
|
||||
append_transcript_delta(&mut active_transcript.entries, "assistant", delta);
|
||||
}
|
||||
RealtimeEvent::HandoffRequested(handoff) => {
|
||||
handoff.active_transcript = std::mem::take(&mut active_transcript.entries);
|
||||
if self.event_parser == RealtimeEventParser::V1 {
|
||||
handoff.active_transcript = std::mem::take(&mut active_transcript.entries);
|
||||
}
|
||||
}
|
||||
RealtimeEvent::SessionUpdated { .. }
|
||||
| RealtimeEvent::AudioOut(_)
|
||||
| RealtimeEvent::ResponseCancelled(_)
|
||||
| RealtimeEvent::ConversationItemAdded(_)
|
||||
| RealtimeEvent::ConversationItemDone { .. }
|
||||
| RealtimeEvent::Error(_) => {}
|
||||
@@ -616,6 +628,8 @@ mod tests {
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeHandoffRequested;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptDelta;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeTranscriptEntry;
|
||||
use codex_protocol::protocol::RealtimeInputAudioSpeechStarted;
|
||||
use codex_protocol::protocol::RealtimeResponseCancelled;
|
||||
use http::HeaderValue;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
@@ -660,6 +674,7 @@ mod tests {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(960),
|
||||
item_id: None,
|
||||
}))
|
||||
);
|
||||
}
|
||||
@@ -809,10 +824,112 @@ mod tests {
|
||||
sample_rate: 24_000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: None,
|
||||
item_id: None,
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_realtime_v2_response_audio_delta_with_item_id() {
|
||||
let payload = json!({
|
||||
"type": "response.audio.delta",
|
||||
"delta": "AQID",
|
||||
"item_id": "item_audio_1"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
||||
Some(RealtimeEvent::AudioOut(RealtimeAudioFrame {
|
||||
data: "AQID".to_string(),
|
||||
sample_rate: 24_000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: None,
|
||||
item_id: Some("item_audio_1".to_string()),
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_realtime_v2_speech_started_event() {
|
||||
let payload = json!({
|
||||
"type": "input_audio_buffer.speech_started",
|
||||
"item_id": "item_input_1"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
||||
Some(RealtimeEvent::InputAudioSpeechStarted(
|
||||
RealtimeInputAudioSpeechStarted {
|
||||
item_id: Some("item_input_1".to_string()),
|
||||
}
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_realtime_v2_response_cancelled_event() {
|
||||
let payload = json!({
|
||||
"type": "response.cancelled",
|
||||
"response": {"id": "resp_cancelled_1"}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
||||
Some(RealtimeEvent::ResponseCancelled(
|
||||
RealtimeResponseCancelled {
|
||||
response_id: Some("resp_cancelled_1".to_string()),
|
||||
}
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_realtime_v2_response_done_handoff_event() {
|
||||
let payload = json!({
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"output": [{
|
||||
"id": "item_123",
|
||||
"type": "function_call",
|
||||
"name": "codex",
|
||||
"call_id": "call_123",
|
||||
"arguments": "{\"prompt\":\"delegate from done\"}"
|
||||
}]
|
||||
}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
||||
Some(RealtimeEvent::HandoffRequested(RealtimeHandoffRequested {
|
||||
handoff_id: "call_123".to_string(),
|
||||
item_id: "item_123".to_string(),
|
||||
input_transcript: "delegate from done".to_string(),
|
||||
active_transcript: Vec::new(),
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_realtime_v2_response_created_event() {
|
||||
let payload = json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": "resp_created_1"}
|
||||
})
|
||||
.to_string();
|
||||
|
||||
assert_eq!(
|
||||
parse_realtime_event(payload.as_str(), RealtimeEventParser::RealtimeV2),
|
||||
Some(RealtimeEvent::ConversationItemAdded(json!({
|
||||
"type": "response.created",
|
||||
"response": {"id": "resp_created_1"}
|
||||
})))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_request_headers_matches_http_precedence() {
|
||||
let mut provider_headers = HeaderMap::new();
|
||||
@@ -1058,7 +1175,10 @@ mod tests {
|
||||
let fourth_json: Value = serde_json::from_str(&fourth).expect("json");
|
||||
assert_eq!(fourth_json["type"], "conversation.handoff.append");
|
||||
assert_eq!(fourth_json["handoff_id"], "handoff_1");
|
||||
assert_eq!(fourth_json["output_text"], "hello from codex");
|
||||
assert_eq!(
|
||||
fourth_json["output_text"],
|
||||
"\"Agent Final Message\":\n\nhello from codex"
|
||||
);
|
||||
|
||||
ws.send(Message::Text(
|
||||
json!({
|
||||
@@ -1169,6 +1289,7 @@ mod tests {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(960),
|
||||
item_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("send audio");
|
||||
@@ -1196,6 +1317,7 @@ mod tests {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: None,
|
||||
item_id: None,
|
||||
})
|
||||
);
|
||||
|
||||
@@ -1285,9 +1407,38 @@ mod tests {
|
||||
first_json["session"]["type"],
|
||||
Value::String("realtime".to_string())
|
||||
);
|
||||
assert_eq!(first_json["session"]["output_modalities"], json!(["audio"]));
|
||||
assert_eq!(
|
||||
first_json["session"]["audio"]["input"]["format"],
|
||||
json!({
|
||||
"type": "audio/pcm",
|
||||
"rate": 24_000,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["audio"]["input"]["noise_reduction"],
|
||||
json!({
|
||||
"type": "near_field",
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["audio"]["input"]["turn_detection"],
|
||||
json!({
|
||||
"type": "server_vad",
|
||||
"interrupt_response": true,
|
||||
"create_response": true,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["audio"]["output"]["format"],
|
||||
json!({
|
||||
"type": "audio/pcm",
|
||||
"rate": 24_000,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["audio"]["output"]["voice"],
|
||||
Value::String("alloy".to_string())
|
||||
Value::String("marin".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["tools"][0]["type"],
|
||||
@@ -1301,6 +1452,10 @@ mod tests {
|
||||
first_json["session"]["tools"][0]["parameters"]["required"],
|
||||
json!(["prompt"])
|
||||
);
|
||||
assert_eq!(
|
||||
first_json["session"]["tool_choice"],
|
||||
Value::String("auto".to_string())
|
||||
);
|
||||
|
||||
ws.send(Message::Text(
|
||||
json!({
|
||||
@@ -1354,7 +1509,7 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
third_json["item"]["output"],
|
||||
Value::String("delegated result".to_string())
|
||||
Value::String("\"Agent Final Message\":\n\ndelegated result".to_string())
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1511,6 +1666,7 @@ mod tests {
|
||||
sample_rate: 24_000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(480),
|
||||
item_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("send audio");
|
||||
@@ -1690,6 +1846,7 @@ mod tests {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(960),
|
||||
item_id: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -12,7 +12,7 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
|
||||
|
||||
pub(super) const REALTIME_AUDIO_SAMPLE_RATE: u32 = 24_000;
|
||||
pub(super) const REALTIME_AUDIO_FORMAT: &str = "audio/pcm";
|
||||
const AGENT_FINAL_MESSAGE_PREFIX: &str = "\"Agent Final Message\":\n\n";
|
||||
|
||||
pub(super) fn normalized_session_mode(
|
||||
event_parser: RealtimeEventParser,
|
||||
@@ -39,6 +39,7 @@ pub(super) fn conversation_handoff_append_message(
|
||||
handoff_id: String,
|
||||
output_text: String,
|
||||
) -> RealtimeOutboundMessage {
|
||||
let output_text = format!("{AGENT_FINAL_MESSAGE_PREFIX}{output_text}");
|
||||
match event_parser {
|
||||
RealtimeEventParser::V1 => v1_conversation_handoff_append_message(handoff_id, output_text),
|
||||
RealtimeEventParser::RealtimeV2 => {
|
||||
|
||||
@@ -1,25 +1,27 @@
|
||||
use crate::endpoint::realtime_websocket::methods_common::REALTIME_AUDIO_FORMAT;
|
||||
use crate::endpoint::realtime_websocket::methods_common::REALTIME_AUDIO_SAMPLE_RATE;
|
||||
use crate::endpoint::realtime_websocket::protocol::AudioFormatType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationContentType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemContent;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemPayload;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationMessageItem;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationRole;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudio;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioFormat;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioInput;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioOutput;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioVoice;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionType;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
|
||||
|
||||
const REALTIME_V1_SESSION_TYPE: &str = "quicksilver";
|
||||
|
||||
pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage {
|
||||
RealtimeOutboundMessage::ConversationItemCreate {
|
||||
item: ConversationItemPayload::Message(ConversationMessageItem {
|
||||
kind: "message".to_string(),
|
||||
role: "user".to_string(),
|
||||
r#type: ConversationItemType::Message,
|
||||
role: ConversationRole::User,
|
||||
content: vec![ConversationItemContent {
|
||||
kind: "text".to_string(),
|
||||
r#type: ConversationContentType::Text,
|
||||
text,
|
||||
}],
|
||||
}),
|
||||
@@ -38,20 +40,25 @@ pub(super) fn conversation_handoff_append_message(
|
||||
|
||||
pub(super) fn session_update_session(instructions: String) -> SessionUpdateSession {
|
||||
SessionUpdateSession {
|
||||
kind: REALTIME_V1_SESSION_TYPE.to_string(),
|
||||
r#type: SessionType::Quicksilver,
|
||||
instructions: Some(instructions),
|
||||
output_modalities: None,
|
||||
audio: SessionAudio {
|
||||
input: SessionAudioInput {
|
||||
format: SessionAudioFormat {
|
||||
kind: REALTIME_AUDIO_FORMAT.to_string(),
|
||||
r#type: AudioFormatType::AudioPcm,
|
||||
rate: REALTIME_AUDIO_SAMPLE_RATE,
|
||||
},
|
||||
noise_reduction: None,
|
||||
turn_detection: None,
|
||||
},
|
||||
output: Some(SessionAudioOutput {
|
||||
format: None,
|
||||
voice: SessionAudioVoice::Fathom,
|
||||
}),
|
||||
},
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,31 +1,42 @@
|
||||
use crate::endpoint::realtime_websocket::methods_common::REALTIME_AUDIO_FORMAT;
|
||||
use crate::endpoint::realtime_websocket::methods_common::REALTIME_AUDIO_SAMPLE_RATE;
|
||||
use crate::endpoint::realtime_websocket::protocol::AudioFormatType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationContentType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationFunctionCallOutputItem;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemContent;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemPayload;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationItemType;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationMessageItem;
|
||||
use crate::endpoint::realtime_websocket::protocol::ConversationRole;
|
||||
use crate::endpoint::realtime_websocket::protocol::NoiseReductionType;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudio;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioFormat;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioInput;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioOutput;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioOutputFormat;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionAudioVoice;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionFunctionTool;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionNoiseReduction;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionToolType;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionTurnDetection;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionType;
|
||||
use crate::endpoint::realtime_websocket::protocol::SessionUpdateSession;
|
||||
use crate::endpoint::realtime_websocket::protocol::TurnDetectionType;
|
||||
use serde_json::json;
|
||||
|
||||
const REALTIME_V2_SESSION_TYPE: &str = "realtime";
|
||||
const REALTIME_V2_OUTPUT_MODALITY_AUDIO: &str = "audio";
|
||||
const REALTIME_V2_TOOL_CHOICE: &str = "auto";
|
||||
const REALTIME_V2_CODEX_TOOL_NAME: &str = "codex";
|
||||
const REALTIME_V2_CODEX_TOOL_DESCRIPTION: &str = "Delegate work to Codex and return the result.";
|
||||
const REALTIME_V2_CODEX_TOOL_DESCRIPTION: &str = "Delegate a request to Codex and return the final result to the user. Use this as the default action. If the user asks to do something next, later, after this, or once current work finishes, call this tool so the work is actually queued instead of merely promising to do it later.";
|
||||
|
||||
pub(super) fn conversation_item_create_message(text: String) -> RealtimeOutboundMessage {
|
||||
RealtimeOutboundMessage::ConversationItemCreate {
|
||||
item: ConversationItemPayload::Message(ConversationMessageItem {
|
||||
kind: "message".to_string(),
|
||||
role: "user".to_string(),
|
||||
r#type: ConversationItemType::Message,
|
||||
role: ConversationRole::User,
|
||||
content: vec![ConversationItemContent {
|
||||
kind: "input_text".to_string(),
|
||||
r#type: ConversationContentType::InputText,
|
||||
text,
|
||||
}],
|
||||
}),
|
||||
@@ -38,7 +49,7 @@ pub(super) fn conversation_handoff_append_message(
|
||||
) -> RealtimeOutboundMessage {
|
||||
RealtimeOutboundMessage::ConversationItemCreate {
|
||||
item: ConversationItemPayload::FunctionCallOutput(ConversationFunctionCallOutputItem {
|
||||
kind: "function_call_output".to_string(),
|
||||
r#type: ConversationItemType::FunctionCallOutput,
|
||||
call_id: handoff_id,
|
||||
output: output_text,
|
||||
}),
|
||||
@@ -51,21 +62,34 @@ pub(super) fn session_update_session(
|
||||
) -> SessionUpdateSession {
|
||||
match session_mode {
|
||||
RealtimeSessionMode::Conversational => SessionUpdateSession {
|
||||
kind: REALTIME_V2_SESSION_TYPE.to_string(),
|
||||
r#type: SessionType::Realtime,
|
||||
instructions: Some(instructions),
|
||||
output_modalities: Some(vec![REALTIME_V2_OUTPUT_MODALITY_AUDIO.to_string()]),
|
||||
audio: SessionAudio {
|
||||
input: SessionAudioInput {
|
||||
format: SessionAudioFormat {
|
||||
kind: REALTIME_AUDIO_FORMAT.to_string(),
|
||||
r#type: AudioFormatType::AudioPcm,
|
||||
rate: REALTIME_AUDIO_SAMPLE_RATE,
|
||||
},
|
||||
noise_reduction: Some(SessionNoiseReduction {
|
||||
r#type: NoiseReductionType::NearField,
|
||||
}),
|
||||
turn_detection: Some(SessionTurnDetection {
|
||||
r#type: TurnDetectionType::ServerVad,
|
||||
interrupt_response: true,
|
||||
create_response: true,
|
||||
}),
|
||||
},
|
||||
output: Some(SessionAudioOutput {
|
||||
voice: SessionAudioVoice::Alloy,
|
||||
format: Some(SessionAudioOutputFormat {
|
||||
r#type: AudioFormatType::AudioPcm,
|
||||
rate: REALTIME_AUDIO_SAMPLE_RATE,
|
||||
}),
|
||||
voice: SessionAudioVoice::Marin,
|
||||
}),
|
||||
},
|
||||
tools: Some(vec![SessionFunctionTool {
|
||||
kind: "function".to_string(),
|
||||
r#type: SessionToolType::Function,
|
||||
name: REALTIME_V2_CODEX_TOOL_NAME.to_string(),
|
||||
description: REALTIME_V2_CODEX_TOOL_DESCRIPTION.to_string(),
|
||||
parameters: json!({
|
||||
@@ -73,27 +97,32 @@ pub(super) fn session_update_session(
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "Prompt text for the delegated Codex task."
|
||||
"description": "The user request to delegate to Codex."
|
||||
}
|
||||
},
|
||||
"required": ["prompt"],
|
||||
"additionalProperties": false
|
||||
}),
|
||||
}]),
|
||||
tool_choice: Some(REALTIME_V2_TOOL_CHOICE.to_string()),
|
||||
},
|
||||
RealtimeSessionMode::Transcription => SessionUpdateSession {
|
||||
kind: "transcription".to_string(),
|
||||
r#type: SessionType::Transcription,
|
||||
instructions: None,
|
||||
output_modalities: None,
|
||||
audio: SessionAudio {
|
||||
input: SessionAudioInput {
|
||||
format: SessionAudioFormat {
|
||||
kind: REALTIME_AUDIO_FORMAT.to_string(),
|
||||
r#type: AudioFormatType::AudioPcm,
|
||||
rate: REALTIME_AUDIO_SAMPLE_RATE,
|
||||
},
|
||||
noise_reduction: None,
|
||||
turn_detection: None,
|
||||
},
|
||||
output: None,
|
||||
},
|
||||
tools: None,
|
||||
tool_choice: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@ pub(super) enum RealtimeOutboundMessage {
|
||||
handoff_id: String,
|
||||
output_text: String,
|
||||
},
|
||||
#[serde(rename = "response.create")]
|
||||
ResponseCreate,
|
||||
#[serde(rename = "session.update")]
|
||||
SessionUpdate { session: SessionUpdateSession },
|
||||
#[serde(rename = "conversation.item.create")]
|
||||
@@ -48,12 +50,24 @@ pub(super) enum RealtimeOutboundMessage {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionUpdateSession {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) r#type: SessionType,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) instructions: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) output_modalities: Option<Vec<String>>,
|
||||
pub(super) audio: SessionAudio,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) tools: Option<Vec<SessionFunctionTool>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) tool_choice: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum SessionType {
|
||||
Quicksilver,
|
||||
Realtime,
|
||||
Transcription,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
@@ -66,17 +80,29 @@ pub(super) struct SessionAudio {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionAudioInput {
|
||||
pub(super) format: SessionAudioFormat,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) noise_reduction: Option<SessionNoiseReduction>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) turn_detection: Option<SessionTurnDetection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionAudioFormat {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) r#type: AudioFormatType,
|
||||
pub(super) rate: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
pub(super) enum AudioFormatType {
|
||||
#[serde(rename = "audio/pcm")]
|
||||
AudioPcm,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionAudioOutput {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(super) format: Option<SessionAudioOutputFormat>,
|
||||
pub(super) voice: SessionAudioVoice,
|
||||
}
|
||||
|
||||
@@ -84,18 +110,64 @@ pub(super) struct SessionAudioOutput {
|
||||
pub(super) enum SessionAudioVoice {
|
||||
#[serde(rename = "fathom")]
|
||||
Fathom,
|
||||
#[serde(rename = "alloy")]
|
||||
Alloy,
|
||||
#[serde(rename = "marin")]
|
||||
Marin,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionNoiseReduction {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) r#type: NoiseReductionType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum NoiseReductionType {
|
||||
NearField,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionTurnDetection {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) r#type: TurnDetectionType,
|
||||
pub(super) interrupt_response: bool,
|
||||
pub(super) create_response: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum TurnDetectionType {
|
||||
ServerVad,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionAudioOutputFormat {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) r#type: AudioFormatType,
|
||||
pub(super) rate: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ConversationMessageItem {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) role: String,
|
||||
pub(super) r#type: ConversationItemType,
|
||||
pub(super) role: ConversationRole,
|
||||
pub(super) content: Vec<ConversationItemContent>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum ConversationItemType {
|
||||
Message,
|
||||
FunctionCallOutput,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum ConversationRole {
|
||||
User,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub(super) enum ConversationItemPayload {
|
||||
@@ -106,7 +178,7 @@ pub(super) enum ConversationItemPayload {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ConversationFunctionCallOutputItem {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) r#type: ConversationItemType,
|
||||
pub(super) call_id: String,
|
||||
pub(super) output: String,
|
||||
}
|
||||
@@ -114,19 +186,32 @@ pub(super) struct ConversationFunctionCallOutputItem {
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct ConversationItemContent {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) r#type: ConversationContentType,
|
||||
pub(super) text: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum ConversationContentType {
|
||||
Text,
|
||||
InputText,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(super) struct SessionFunctionTool {
|
||||
#[serde(rename = "type")]
|
||||
pub(super) kind: String,
|
||||
pub(super) r#type: SessionToolType,
|
||||
pub(super) name: String,
|
||||
pub(super) description: String,
|
||||
pub(super) parameters: Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub(super) enum SessionToolType {
|
||||
Function,
|
||||
}
|
||||
|
||||
pub(super) fn parse_realtime_event(
|
||||
payload: &str,
|
||||
event_parser: RealtimeEventParser,
|
||||
|
||||
@@ -35,6 +35,7 @@ pub(super) fn parse_realtime_event_v1(payload: &str) -> Option<RealtimeEvent> {
|
||||
.get("samples_per_channel")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|value| u32::try_from(value).ok()),
|
||||
item_id: None,
|
||||
}))
|
||||
}
|
||||
"conversation.input_transcript.delta" => {
|
||||
|
||||
@@ -5,6 +5,8 @@ use crate::endpoint::realtime_websocket::protocol_common::parse_transcript_delta
|
||||
use codex_protocol::protocol::RealtimeAudioFrame;
|
||||
use codex_protocol::protocol::RealtimeEvent;
|
||||
use codex_protocol::protocol::RealtimeHandoffRequested;
|
||||
use codex_protocol::protocol::RealtimeInputAudioSpeechStarted;
|
||||
use codex_protocol::protocol::RealtimeResponseCancelled;
|
||||
use serde_json::Map as JsonMap;
|
||||
use serde_json::Value;
|
||||
use tracing::debug;
|
||||
@@ -19,7 +21,9 @@ pub(super) fn parse_realtime_event_v2(payload: &str) -> Option<RealtimeEvent> {
|
||||
|
||||
match message_type.as_str() {
|
||||
"session.updated" => parse_session_updated_event(&parsed),
|
||||
"response.output_audio.delta" => parse_output_audio_delta_event(&parsed),
|
||||
"response.output_audio.delta" | "response.audio.delta" => {
|
||||
parse_output_audio_delta_event(&parsed)
|
||||
}
|
||||
"conversation.item.input_audio_transcription.delta" => {
|
||||
parse_transcript_delta_event(&parsed, "delta").map(RealtimeEvent::InputTranscriptDelta)
|
||||
}
|
||||
@@ -30,11 +34,37 @@ pub(super) fn parse_realtime_event_v2(payload: &str) -> Option<RealtimeEvent> {
|
||||
"response.output_text.delta" | "response.output_audio_transcript.delta" => {
|
||||
parse_transcript_delta_event(&parsed, "delta").map(RealtimeEvent::OutputTranscriptDelta)
|
||||
}
|
||||
"input_audio_buffer.speech_started" => Some(RealtimeEvent::InputAudioSpeechStarted(
|
||||
RealtimeInputAudioSpeechStarted {
|
||||
item_id: parsed
|
||||
.get("item_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
},
|
||||
)),
|
||||
"conversation.item.added" => parsed
|
||||
.get("item")
|
||||
.cloned()
|
||||
.map(RealtimeEvent::ConversationItemAdded),
|
||||
"conversation.item.done" => parse_conversation_item_done_event(&parsed),
|
||||
"response.created" => Some(RealtimeEvent::ConversationItemAdded(parsed)),
|
||||
"response.done" => parse_response_done_event(parsed),
|
||||
"response.cancelled" => Some(RealtimeEvent::ResponseCancelled(
|
||||
RealtimeResponseCancelled {
|
||||
response_id: parsed
|
||||
.get("response")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|response| response.get("id"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
parsed
|
||||
.get("response_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
}),
|
||||
},
|
||||
)),
|
||||
"error" => parse_error_event(&parsed),
|
||||
_ => {
|
||||
debug!("received unsupported realtime v2 event type: {message_type}, data: {payload}");
|
||||
@@ -67,6 +97,10 @@ fn parse_output_audio_delta_event(parsed: &Value) -> Option<RealtimeEvent> {
|
||||
.get("samples_per_channel")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|value| u32::try_from(value).ok()),
|
||||
item_id: parsed
|
||||
.get("item_id")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -82,6 +116,30 @@ fn parse_conversation_item_done_event(parsed: &Value) -> Option<RealtimeEvent> {
|
||||
.map(|item_id| RealtimeEvent::ConversationItemDone { item_id })
|
||||
}
|
||||
|
||||
fn parse_response_done_event(parsed: Value) -> Option<RealtimeEvent> {
|
||||
if let Some(handoff) = parse_response_done_handoff_requested_event(&parsed) {
|
||||
return Some(handoff);
|
||||
}
|
||||
|
||||
Some(RealtimeEvent::ConversationItemAdded(parsed))
|
||||
}
|
||||
|
||||
fn parse_response_done_handoff_requested_event(parsed: &Value) -> Option<RealtimeEvent> {
|
||||
let item = parsed
|
||||
.get("response")
|
||||
.and_then(Value::as_object)
|
||||
.and_then(|response| response.get("output"))
|
||||
.and_then(Value::as_array)?
|
||||
.iter()
|
||||
.find(|item| {
|
||||
item.get("type").and_then(Value::as_str) == Some("function_call")
|
||||
&& item.get("name").and_then(Value::as_str) == Some(CODEX_TOOL_NAME)
|
||||
})?
|
||||
.as_object()?;
|
||||
|
||||
parse_handoff_requested_event(item)
|
||||
}
|
||||
|
||||
fn parse_handoff_requested_event(item: &JsonMap<String, Value>) -> Option<RealtimeEvent> {
|
||||
let item_type = item.get("type").and_then(Value::as_str);
|
||||
let item_name = item.get("name").and_then(Value::as_str);
|
||||
|
||||
@@ -23,7 +23,10 @@ pub use crate::common::ResponseCreateWsRequest;
|
||||
pub use crate::common::ResponseEvent;
|
||||
pub use crate::common::ResponseStream;
|
||||
pub use crate::common::ResponsesApiRequest;
|
||||
pub use crate::common::WS_REQUEST_HEADER_TRACEPARENT_CLIENT_METADATA_KEY;
|
||||
pub use crate::common::WS_REQUEST_HEADER_TRACESTATE_CLIENT_METADATA_KEY;
|
||||
pub use crate::common::create_text_param_for_request;
|
||||
pub use crate::common::response_create_client_metadata;
|
||||
pub use crate::endpoint::compact::CompactClient;
|
||||
pub use crate::endpoint::memories::MemoriesClient;
|
||||
pub use crate::endpoint::models::ModelsClient;
|
||||
|
||||
@@ -93,7 +93,6 @@ async fn models_client_hits_models_endpoint() {
|
||||
effective_context_window_percent: 95,
|
||||
experimental_supported_tools: Vec::new(),
|
||||
input_modalities: default_input_modalities(),
|
||||
prefer_websockets: false,
|
||||
used_fallback_model_metadata: false,
|
||||
supports_search_tool: false,
|
||||
}],
|
||||
|
||||
@@ -170,6 +170,7 @@ async fn realtime_ws_e2e_session_create_and_event_flow() {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(960),
|
||||
item_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("send audio");
|
||||
@@ -186,6 +187,7 @@ async fn realtime_ws_e2e_session_create_and_event_flow() {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: None,
|
||||
item_id: None,
|
||||
})
|
||||
);
|
||||
|
||||
@@ -254,6 +256,7 @@ async fn realtime_ws_e2e_send_while_next_event_waits() {
|
||||
sample_rate: 48000,
|
||||
num_channels: 1,
|
||||
samples_per_channel: Some(960),
|
||||
item_id: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
|
||||
Reference in New Issue
Block a user