mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
Seed realtime V3 sessions with initial text items (#34067)
## What changed - Add an optional `initialItems` field to `thread/realtime/start` for seeding realtime V3 session history with `user`, `developer`, and `assistant` text. - Encode the items in Frameless Bidi session bootstrap payloads for websocket and WebRTC transports while preserving existing payloads when the list is omitted or empty. - Reject non-empty initial items for V1 and V2, more than 128 items, any item over 8,192 estimated tokens, or a combined total over 8,192 estimated tokens. ## Testing - Cover protocol serialization, app-server forwarding, websocket and WebRTC payloads, empty-list compatibility, version checks, and size limits. GitOrigin-RevId: 09af367f9a07543e3f8d59b7fa5387f932dc5a6a
This commit is contained in:
committed by
copyberry
parent
35eaf3ffb0
commit
312caf176a
@@ -305,6 +305,8 @@ mod tests {
|
||||
use codex_client::Response;
|
||||
use codex_client::StreamResponse;
|
||||
use codex_client::TransportError;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
use codex_protocol::protocol::RealtimeVoice;
|
||||
use http::StatusCode;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -386,6 +388,7 @@ mod tests {
|
||||
fn realtime_session_config(session_id: &str) -> RealtimeSessionConfig {
|
||||
RealtimeSessionConfig {
|
||||
instructions: "hi".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("gpt-realtime".to_string()),
|
||||
session_id: Some(session_id.to_string()),
|
||||
event_parser: RealtimeEventParser::V1,
|
||||
@@ -697,12 +700,20 @@ mod tests {
|
||||
provider("https://chatgpt.com/backend-api/codex"),
|
||||
Arc::new(DummyAuth),
|
||||
);
|
||||
let mut session_config = frameless_bidi_session_config("sess-backend");
|
||||
session_config.initial_items = vec![
|
||||
ConversationTextParams {
|
||||
text: "Remember this.".to_string(),
|
||||
role: ConversationTextRole::Developer,
|
||||
},
|
||||
ConversationTextParams {
|
||||
text: "Understood.".to_string(),
|
||||
role: ConversationTextRole::Assistant,
|
||||
},
|
||||
];
|
||||
|
||||
let response = client
|
||||
.create_with_session(
|
||||
"v=offer\r\n".to_string(),
|
||||
frameless_bidi_session_config("sess-backend"),
|
||||
)
|
||||
.create_with_session("v=offer\r\n".to_string(), session_config)
|
||||
.await
|
||||
.expect("request should succeed");
|
||||
|
||||
@@ -718,6 +729,21 @@ mod tests {
|
||||
};
|
||||
assert_eq!(body["session"]["delegation"]["type"], "client");
|
||||
assert!(body["session"].get("id").is_none());
|
||||
assert_eq!(
|
||||
body["session"]["initial_items"],
|
||||
serde_json::json!([
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [{"type": "input_text", "text": "Remember this."}],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Understood."}],
|
||||
},
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::error::ApiError;
|
||||
use crate::provider::Provider;
|
||||
use codex_client::backoff;
|
||||
use codex_http_client::maybe_build_rustls_client_config_with_custom_ca;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
use codex_protocol::protocol::RealtimeTranscriptDelta;
|
||||
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
|
||||
@@ -378,6 +379,7 @@ impl RealtimeWebsocketWriter {
|
||||
pub async fn send_session_update(
|
||||
&self,
|
||||
instructions: String,
|
||||
initial_items: Vec<ConversationTextParams>,
|
||||
session_mode: RealtimeSessionMode,
|
||||
output_modality: RealtimeOutputModality,
|
||||
voice: RealtimeVoice,
|
||||
@@ -386,6 +388,7 @@ impl RealtimeWebsocketWriter {
|
||||
let message = session_update_message(
|
||||
self.event_parser,
|
||||
instructions,
|
||||
initial_items,
|
||||
session_mode,
|
||||
output_modality,
|
||||
voice,
|
||||
@@ -841,6 +844,7 @@ impl RealtimeWebsocketClient {
|
||||
.writer
|
||||
.send_session_update(
|
||||
config.instructions,
|
||||
config.initial_items,
|
||||
config.session_mode,
|
||||
config.output_modality,
|
||||
config.voice,
|
||||
@@ -1965,6 +1969,7 @@ mod tests {
|
||||
.connect(
|
||||
RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("realtime-test-model".to_string()),
|
||||
session_id: Some("conv_1".to_string()),
|
||||
event_parser: RealtimeEventParser::V1,
|
||||
@@ -2289,6 +2294,7 @@ mod tests {
|
||||
.connect(
|
||||
RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("realtime-test-model".to_string()),
|
||||
session_id: Some("conv_1".to_string()),
|
||||
event_parser: RealtimeEventParser::RealtimeV2,
|
||||
@@ -2414,6 +2420,7 @@ mod tests {
|
||||
.connect(
|
||||
RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("realtime-test-model".to_string()),
|
||||
session_id: Some("conv_1".to_string()),
|
||||
event_parser: RealtimeEventParser::RealtimeV2,
|
||||
@@ -2518,6 +2525,7 @@ mod tests {
|
||||
.connect(
|
||||
RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("realtime-test-model".to_string()),
|
||||
session_id: Some("conv_1".to_string()),
|
||||
event_parser: RealtimeEventParser::V1,
|
||||
@@ -2608,6 +2616,7 @@ mod tests {
|
||||
.connect(
|
||||
RealtimeSessionConfig {
|
||||
instructions: "backend prompt".to_string(),
|
||||
initial_items: Vec::new(),
|
||||
model: Some("realtime-test-model".to_string()),
|
||||
session_id: Some("conv_1".to_string()),
|
||||
event_parser: RealtimeEventParser::V1,
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeWireAdapter;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
use serde_json::Result as JsonResult;
|
||||
use serde_json::Value;
|
||||
@@ -113,6 +114,7 @@ pub(super) fn conversation_function_call_output_message(
|
||||
pub(super) fn session_update_message(
|
||||
wire_adapter: RealtimeWireAdapter,
|
||||
instructions: String,
|
||||
initial_items: Vec<ConversationTextParams>,
|
||||
session_mode: RealtimeSessionMode,
|
||||
output_modality: RealtimeOutputModality,
|
||||
voice: RealtimeVoice,
|
||||
@@ -122,7 +124,9 @@ pub(super) fn session_update_message(
|
||||
RealtimeWireAdapter::V1 => RealtimeOutboundMessage::SessionUpdate {
|
||||
session: v1_session_update_session(instructions, voice),
|
||||
},
|
||||
RealtimeWireAdapter::FramelessBidi => frameless_session_update_message(instructions, voice),
|
||||
RealtimeWireAdapter::FramelessBidi => {
|
||||
frameless_session_update_message(instructions, initial_items, voice)
|
||||
}
|
||||
RealtimeWireAdapter::RealtimeV2 => RealtimeOutboundMessage::SessionUpdate {
|
||||
session: v2_session_update_session(instructions, session_mode, output_modality, voice),
|
||||
},
|
||||
@@ -151,6 +155,7 @@ pub fn session_update_session_json(config: RealtimeSessionConfig) -> JsonResult<
|
||||
RealtimeWireAdapter::FramelessBidi => Ok(frameless_session_json(
|
||||
config.model,
|
||||
config.instructions,
|
||||
config.initial_items,
|
||||
config.voice,
|
||||
)),
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ use crate::endpoint::realtime_websocket::protocol::FramelessInputTextContent;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
|
||||
@@ -32,16 +34,18 @@ pub(super) fn session_context_append_message(
|
||||
|
||||
pub(super) fn session_update_message(
|
||||
instructions: String,
|
||||
initial_items: Vec<ConversationTextParams>,
|
||||
voice: RealtimeVoice,
|
||||
) -> RealtimeOutboundMessage {
|
||||
RealtimeOutboundMessage::FramelessSessionUpdate {
|
||||
session: session_json(/*model*/ None, instructions, voice),
|
||||
session: session_json(/*model*/ None, instructions, initial_items, voice),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn session_json(
|
||||
model: Option<String>,
|
||||
instructions: String,
|
||||
initial_items: Vec<ConversationTextParams>,
|
||||
voice: RealtimeVoice,
|
||||
) -> Value {
|
||||
let mut session = json!({
|
||||
@@ -58,6 +62,29 @@ pub(super) fn session_json(
|
||||
if let Some(model) = model {
|
||||
session["model"] = Value::String(model);
|
||||
}
|
||||
if !initial_items.is_empty() {
|
||||
session["initial_items"] = Value::Array(
|
||||
initial_items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let content_type = match item.role {
|
||||
ConversationTextRole::User | ConversationTextRole::Developer => {
|
||||
"input_text"
|
||||
}
|
||||
ConversationTextRole::Assistant => "output_text",
|
||||
};
|
||||
json!({
|
||||
"type": "message",
|
||||
"role": item.role,
|
||||
"content": [{
|
||||
"type": content_type,
|
||||
"text": item.text,
|
||||
}],
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
session
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
use super::CONTEXT_APPEND_MAX_BYTES;
|
||||
use super::context_append_chunks;
|
||||
use super::session_json;
|
||||
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn context_append_chunks_preserve_text_within_wire_limit() {
|
||||
@@ -13,3 +19,82 @@ fn context_append_chunks_preserve_text_within_wire_limit() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_json_omits_initial_items_when_empty() {
|
||||
let session = session_json(
|
||||
Some("gpt-live".to_string()),
|
||||
"instructions".to_string(),
|
||||
Vec::new(),
|
||||
RealtimeVoice::Marin,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
session,
|
||||
json!({
|
||||
"model": "gpt-live",
|
||||
"instructions": "instructions",
|
||||
"audio": {
|
||||
"output": {
|
||||
"voice": "marin",
|
||||
},
|
||||
},
|
||||
"delegation": {
|
||||
"type": "client",
|
||||
},
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_json_encodes_role_bearing_initial_items() {
|
||||
let session = session_json(
|
||||
Some("gpt-live".to_string()),
|
||||
"instructions".to_string(),
|
||||
vec![
|
||||
ConversationTextParams {
|
||||
text: "Remember this.".to_string(),
|
||||
role: ConversationTextRole::Developer,
|
||||
},
|
||||
ConversationTextParams {
|
||||
text: "What do you remember?".to_string(),
|
||||
role: ConversationTextRole::User,
|
||||
},
|
||||
ConversationTextParams {
|
||||
text: "I remember.".to_string(),
|
||||
role: ConversationTextRole::Assistant,
|
||||
},
|
||||
],
|
||||
RealtimeVoice::Marin,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
session["initial_items"],
|
||||
json!([
|
||||
{
|
||||
"type": "message",
|
||||
"role": "developer",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": "Remember this.",
|
||||
}],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{
|
||||
"type": "input_text",
|
||||
"text": "What do you remember?",
|
||||
}],
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{
|
||||
"type": "output_text",
|
||||
"text": "I remember.",
|
||||
}],
|
||||
},
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
|
||||
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
|
||||
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
|
||||
use codex_protocol::protocol::ConversationTextParams;
|
||||
use codex_protocol::protocol::ConversationTextRole;
|
||||
pub use codex_protocol::protocol::RealtimeAudioFrame;
|
||||
pub use codex_protocol::protocol::RealtimeEvent;
|
||||
@@ -36,6 +37,7 @@ pub enum RealtimeContextAppendChannel {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RealtimeSessionConfig {
|
||||
pub instructions: String,
|
||||
pub initial_items: Vec<ConversationTextParams>,
|
||||
pub model: Option<String>,
|
||||
pub session_id: Option<String>,
|
||||
pub event_parser: RealtimeEventParser,
|
||||
|
||||
Reference in New Issue
Block a user