Seed realtime V3 sessions with initial text items (#34067)

## What changed

- Add an optional `initialItems` field to `thread/realtime/start` for seeding
  realtime V3 session history with `user`, `developer`, and `assistant` text.
- Encode the items in Frameless Bidi session bootstrap payloads for websocket
  and WebRTC transports while preserving existing payloads when the list is
  omitted or empty.
- Reject non-empty initial items for V1 and V2, more than 128 items, any item
  over 8,192 estimated tokens, or a combined total over 8,192 estimated tokens.

## Testing

- Cover protocol serialization, app-server forwarding, websocket and WebRTC
  payloads, empty-list compatibility, version checks, and size limits.

GitOrigin-RevId: 09af367f9a07543e3f8d59b7fa5387f932dc5a6a
This commit is contained in:
jiayuhuang-openai
2026-07-18 20:59:57 +00:00
committed by copyberry
parent 35eaf3ffb0
commit 312caf176a
25 changed files with 657 additions and 9 deletions
@@ -305,6 +305,8 @@ mod tests {
use codex_client::Response;
use codex_client::StreamResponse;
use codex_client::TransportError;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
use codex_protocol::protocol::RealtimeVoice;
use http::StatusCode;
use pretty_assertions::assert_eq;
@@ -386,6 +388,7 @@ mod tests {
fn realtime_session_config(session_id: &str) -> RealtimeSessionConfig {
RealtimeSessionConfig {
instructions: "hi".to_string(),
initial_items: Vec::new(),
model: Some("gpt-realtime".to_string()),
session_id: Some(session_id.to_string()),
event_parser: RealtimeEventParser::V1,
@@ -697,12 +700,20 @@ mod tests {
provider("https://chatgpt.com/backend-api/codex"),
Arc::new(DummyAuth),
);
let mut session_config = frameless_bidi_session_config("sess-backend");
session_config.initial_items = vec![
ConversationTextParams {
text: "Remember this.".to_string(),
role: ConversationTextRole::Developer,
},
ConversationTextParams {
text: "Understood.".to_string(),
role: ConversationTextRole::Assistant,
},
];
let response = client
.create_with_session(
"v=offer\r\n".to_string(),
frameless_bidi_session_config("sess-backend"),
)
.create_with_session("v=offer\r\n".to_string(), session_config)
.await
.expect("request should succeed");
@@ -718,6 +729,21 @@ mod tests {
};
assert_eq!(body["session"]["delegation"]["type"], "client");
assert!(body["session"].get("id").is_none());
assert_eq!(
body["session"]["initial_items"],
serde_json::json!([
{
"type": "message",
"role": "developer",
"content": [{"type": "input_text", "text": "Remember this."}],
},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Understood."}],
},
])
);
}
#[tokio::test]
@@ -23,6 +23,7 @@ use crate::error::ApiError;
use crate::provider::Provider;
use codex_client::backoff;
use codex_http_client::maybe_build_rustls_client_config_with_custom_ca;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
use codex_protocol::protocol::RealtimeTranscriptDelta;
use codex_utils_rustls_provider::ensure_rustls_crypto_provider;
@@ -378,6 +379,7 @@ impl RealtimeWebsocketWriter {
pub async fn send_session_update(
&self,
instructions: String,
initial_items: Vec<ConversationTextParams>,
session_mode: RealtimeSessionMode,
output_modality: RealtimeOutputModality,
voice: RealtimeVoice,
@@ -386,6 +388,7 @@ impl RealtimeWebsocketWriter {
let message = session_update_message(
self.event_parser,
instructions,
initial_items,
session_mode,
output_modality,
voice,
@@ -841,6 +844,7 @@ impl RealtimeWebsocketClient {
.writer
.send_session_update(
config.instructions,
config.initial_items,
config.session_mode,
config.output_modality,
config.voice,
@@ -1965,6 +1969,7 @@ mod tests {
.connect(
RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
initial_items: Vec::new(),
model: Some("realtime-test-model".to_string()),
session_id: Some("conv_1".to_string()),
event_parser: RealtimeEventParser::V1,
@@ -2289,6 +2294,7 @@ mod tests {
.connect(
RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
initial_items: Vec::new(),
model: Some("realtime-test-model".to_string()),
session_id: Some("conv_1".to_string()),
event_parser: RealtimeEventParser::RealtimeV2,
@@ -2414,6 +2420,7 @@ mod tests {
.connect(
RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
initial_items: Vec::new(),
model: Some("realtime-test-model".to_string()),
session_id: Some("conv_1".to_string()),
event_parser: RealtimeEventParser::RealtimeV2,
@@ -2518,6 +2525,7 @@ mod tests {
.connect(
RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
initial_items: Vec::new(),
model: Some("realtime-test-model".to_string()),
session_id: Some("conv_1".to_string()),
event_parser: RealtimeEventParser::V1,
@@ -2608,6 +2616,7 @@ mod tests {
.connect(
RealtimeSessionConfig {
instructions: "backend prompt".to_string(),
initial_items: Vec::new(),
model: Some("realtime-test-model".to_string()),
session_id: Some("conv_1".to_string()),
event_parser: RealtimeEventParser::V1,
@@ -17,6 +17,7 @@ use crate::endpoint::realtime_websocket::protocol::RealtimeSessionConfig;
use crate::endpoint::realtime_websocket::protocol::RealtimeSessionMode;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use crate::endpoint::realtime_websocket::protocol::RealtimeWireAdapter;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
use serde_json::Result as JsonResult;
use serde_json::Value;
@@ -113,6 +114,7 @@ pub(super) fn conversation_function_call_output_message(
pub(super) fn session_update_message(
wire_adapter: RealtimeWireAdapter,
instructions: String,
initial_items: Vec<ConversationTextParams>,
session_mode: RealtimeSessionMode,
output_modality: RealtimeOutputModality,
voice: RealtimeVoice,
@@ -122,7 +124,9 @@ pub(super) fn session_update_message(
RealtimeWireAdapter::V1 => RealtimeOutboundMessage::SessionUpdate {
session: v1_session_update_session(instructions, voice),
},
RealtimeWireAdapter::FramelessBidi => frameless_session_update_message(instructions, voice),
RealtimeWireAdapter::FramelessBidi => {
frameless_session_update_message(instructions, initial_items, voice)
}
RealtimeWireAdapter::RealtimeV2 => RealtimeOutboundMessage::SessionUpdate {
session: v2_session_update_session(instructions, session_mode, output_modality, voice),
},
@@ -151,6 +155,7 @@ pub fn session_update_session_json(config: RealtimeSessionConfig) -> JsonResult<
RealtimeWireAdapter::FramelessBidi => Ok(frameless_session_json(
config.model,
config.instructions,
config.initial_items,
config.voice,
)),
}
@@ -3,6 +3,8 @@ use crate::endpoint::realtime_websocket::protocol::FramelessInputTextContent;
use crate::endpoint::realtime_websocket::protocol::RealtimeContextAppendChannel;
use crate::endpoint::realtime_websocket::protocol::RealtimeOutboundMessage;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
use serde_json::Value;
use serde_json::json;
@@ -32,16 +34,18 @@ pub(super) fn session_context_append_message(
pub(super) fn session_update_message(
instructions: String,
initial_items: Vec<ConversationTextParams>,
voice: RealtimeVoice,
) -> RealtimeOutboundMessage {
RealtimeOutboundMessage::FramelessSessionUpdate {
session: session_json(/*model*/ None, instructions, voice),
session: session_json(/*model*/ None, instructions, initial_items, voice),
}
}
pub(super) fn session_json(
model: Option<String>,
instructions: String,
initial_items: Vec<ConversationTextParams>,
voice: RealtimeVoice,
) -> Value {
let mut session = json!({
@@ -58,6 +62,29 @@ pub(super) fn session_json(
if let Some(model) = model {
session["model"] = Value::String(model);
}
if !initial_items.is_empty() {
session["initial_items"] = Value::Array(
initial_items
.into_iter()
.map(|item| {
let content_type = match item.role {
ConversationTextRole::User | ConversationTextRole::Developer => {
"input_text"
}
ConversationTextRole::Assistant => "output_text",
};
json!({
"type": "message",
"role": item.role,
"content": [{
"type": content_type,
"text": item.text,
}],
})
})
.collect(),
);
}
session
}
@@ -1,5 +1,11 @@
use super::CONTEXT_APPEND_MAX_BYTES;
use super::context_append_chunks;
use super::session_json;
use crate::endpoint::realtime_websocket::protocol::RealtimeVoice;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
use pretty_assertions::assert_eq;
use serde_json::json;
#[test]
fn context_append_chunks_preserve_text_within_wire_limit() {
@@ -13,3 +19,82 @@ fn context_append_chunks_preserve_text_within_wire_limit() {
);
}
}
#[test]
fn session_json_omits_initial_items_when_empty() {
let session = session_json(
Some("gpt-live".to_string()),
"instructions".to_string(),
Vec::new(),
RealtimeVoice::Marin,
);
assert_eq!(
session,
json!({
"model": "gpt-live",
"instructions": "instructions",
"audio": {
"output": {
"voice": "marin",
},
},
"delegation": {
"type": "client",
},
})
);
}
#[test]
fn session_json_encodes_role_bearing_initial_items() {
let session = session_json(
Some("gpt-live".to_string()),
"instructions".to_string(),
vec![
ConversationTextParams {
text: "Remember this.".to_string(),
role: ConversationTextRole::Developer,
},
ConversationTextParams {
text: "What do you remember?".to_string(),
role: ConversationTextRole::User,
},
ConversationTextParams {
text: "I remember.".to_string(),
role: ConversationTextRole::Assistant,
},
],
RealtimeVoice::Marin,
);
assert_eq!(
session["initial_items"],
json!([
{
"type": "message",
"role": "developer",
"content": [{
"type": "input_text",
"text": "Remember this.",
}],
},
{
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "What do you remember?",
}],
},
{
"type": "message",
"role": "assistant",
"content": [{
"type": "output_text",
"text": "I remember.",
}],
},
])
);
}
@@ -1,6 +1,7 @@
use crate::endpoint::realtime_websocket::protocol_frameless_bidi::parse_frameless_bidi_event;
use crate::endpoint::realtime_websocket::protocol_v1::parse_realtime_event_v1;
use crate::endpoint::realtime_websocket::protocol_v2::parse_realtime_event_v2;
use codex_protocol::protocol::ConversationTextParams;
use codex_protocol::protocol::ConversationTextRole;
pub use codex_protocol::protocol::RealtimeAudioFrame;
pub use codex_protocol::protocol::RealtimeEvent;
@@ -36,6 +37,7 @@ pub enum RealtimeContextAppendChannel {
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RealtimeSessionConfig {
pub instructions: String,
pub initial_items: Vec<ConversationTextParams>,
pub model: Option<String>,
pub session_id: Option<String>,
pub event_parser: RealtimeEventParser,