mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
Support attaching to existing realtime calls (#39876)
## Why Clients that create and negotiate realtime calls themselves need Codex to join those calls without creating another call or overwriting the client-owned session configuration. ## What changed - Add the `existingCall` transport to `thread/realtime/start`, accepting a client-provided `callId` and optional `realtimeSessionId`. - Attach Codex over the call's sideband WebSocket without SDP negotiation or a session update, and reconnect that sideband while preserving transcript state. - Default existing calls to realtime v1, support v1 and v3, and reject v2 or options that would reconfigure the client-owned session. - Encode call IDs as a single URL path segment for v3 sideband connections. ## Testing Add protocol, app-server, core, reconnection, authentication-header, and URL encoding coverage for existing-call attachments. GitOrigin-RevId: 2046b70566f98efa55381bb1e461ea3cc68256c5
This commit is contained in:
@@ -1079,6 +1079,27 @@ Then send `offer.sdp` to app-server. Core uses `experimental_realtime_ws_backend
|
||||
} }
|
||||
```
|
||||
|
||||
Clients that create and negotiate the realtime call themselves can instead pass its call ID:
|
||||
|
||||
```json
|
||||
{ "method": "thread/realtime/start", "id": 41, "params": {
|
||||
"threadId": "thr_123",
|
||||
"outputModality": "audio",
|
||||
"version": "v3",
|
||||
"realtimeSessionId": "sess_123",
|
||||
"transport": { "type": "existingCall", "callId": "rtc_123" }
|
||||
} }
|
||||
{ "id": 41, "result": {} }
|
||||
```
|
||||
|
||||
The existing-call transport attaches Codex to the call over its sideband WebSocket without creating
|
||||
another call or emitting `thread/realtime/sdp`. The client owns the SDP negotiation and the initial
|
||||
realtime session configuration. Codex startup context is disabled by default for existing calls;
|
||||
`includeStartupContext: true`, `prompt`, nonempty `initialItems`, `model`, `voice`, and
|
||||
`delegationAckFiller` are rejected because they would change the client-owned session. Supply
|
||||
`realtimeSessionId` when the upstream session ID is known; otherwise the
|
||||
`thread/realtime/started` notification reports `realtimeSessionId: null`.
|
||||
|
||||
Omit `prompt` to use Codex's default realtime backend prompt. Send `prompt: null` or
|
||||
`prompt: ""` when the session should start without that default backend prompt.
|
||||
Pass `realtimeStartInstructions` to provide the developer instructions given to
|
||||
|
||||
@@ -1082,6 +1082,36 @@ impl TurnRequestProcessor {
|
||||
request_id: &ConnectionRequestId,
|
||||
params: ThreadRealtimeStartParams,
|
||||
) -> Result<Option<ThreadRealtimeStartResponse>, JSONRPCErrorError> {
|
||||
let attaches_existing_call = matches!(
|
||||
¶ms.transport,
|
||||
Some(ThreadRealtimeStartTransport::ExistingCall { .. })
|
||||
);
|
||||
if attaches_existing_call {
|
||||
let unsupported_option = if params.include_startup_context == Some(true) {
|
||||
Some("includeStartupContext")
|
||||
} else if params.prompt.is_some() {
|
||||
Some("prompt")
|
||||
} else if params
|
||||
.initial_items
|
||||
.as_ref()
|
||||
.is_some_and(|items| !items.is_empty())
|
||||
{
|
||||
Some("initialItems")
|
||||
} else if params.model.is_some() {
|
||||
Some("model")
|
||||
} else if params.voice.is_some() {
|
||||
Some("voice")
|
||||
} else if params.delegation_ack_filler.is_some() {
|
||||
Some("delegationAckFiller")
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(option) = unsupported_option {
|
||||
return Err(invalid_request(format!(
|
||||
"existingCall transport does not support {option}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
let Some((_, thread)) = self
|
||||
.prepare_realtime_conversation_thread(request_id, ¶ms.thread_id)
|
||||
.await?
|
||||
@@ -1104,7 +1134,9 @@ impl TurnRequestProcessor {
|
||||
.codex_response_handoff_channel_prefixes,
|
||||
model: params.model,
|
||||
output_modality: params.output_modality,
|
||||
include_startup_context: params.include_startup_context.unwrap_or(true),
|
||||
include_startup_context: params
|
||||
.include_startup_context
|
||||
.unwrap_or(!attaches_existing_call),
|
||||
initial_items: params
|
||||
.initial_items
|
||||
.unwrap_or_default()
|
||||
@@ -1125,6 +1157,9 @@ impl TurnRequestProcessor {
|
||||
ThreadRealtimeStartTransport::Webrtc { sdp } => {
|
||||
ConversationStartTransport::Webrtc { sdp }
|
||||
}
|
||||
ThreadRealtimeStartTransport::ExistingCall { call_id } => {
|
||||
ConversationStartTransport::ExistingCall { call_id }
|
||||
}
|
||||
}),
|
||||
version: params.version,
|
||||
voice: params.voice,
|
||||
|
||||
@@ -68,6 +68,7 @@ use std::sync::Mutex;
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
use tempfile::TempDir;
|
||||
use test_case::test_case;
|
||||
use tokio::time::timeout;
|
||||
use wiremock::Match;
|
||||
use wiremock::Mock;
|
||||
@@ -1682,6 +1683,191 @@ async fn webrtc_v1_start_posts_offer_returns_sdp_and_joins_sideband() -> Result<
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case(
|
||||
None,
|
||||
None,
|
||||
RealtimeConversationVersion::V1,
|
||||
"/v1/realtime?intent=quicksilver&call_id=rtc_existing";
|
||||
"defaults to v1"
|
||||
)]
|
||||
#[test_case(
|
||||
Some(RealtimeConversationVersion::V3),
|
||||
Some("sess_client_owned"),
|
||||
RealtimeConversationVersion::V3,
|
||||
"/v1/live/rtc_existing";
|
||||
"supports v3"
|
||||
)]
|
||||
#[test_case(
|
||||
Some(RealtimeConversationVersion::V2),
|
||||
None,
|
||||
RealtimeConversationVersion::V2,
|
||||
"";
|
||||
"rejects v2"
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn existing_call_attaches_without_reinitializing_the_client_session(
|
||||
version: Option<RealtimeConversationVersion>,
|
||||
realtime_session_id: Option<&str>,
|
||||
expected_version: RealtimeConversationVersion,
|
||||
expected_handshake_uri: &str,
|
||||
) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let mut harness = RealtimeE2eHarness::new(
|
||||
RealtimeTestVersion::V1,
|
||||
no_main_loop_responses(),
|
||||
realtime_sideband(vec![open_realtime_sideband_connection(vec![vec![]])]),
|
||||
)
|
||||
.await?;
|
||||
let request_id = harness
|
||||
.mcp
|
||||
.send_thread_realtime_start_request(ThreadRealtimeStartParams {
|
||||
thread_id: harness.thread_id.clone(),
|
||||
client_managed_handoffs: None,
|
||||
delegation_ack_filler: None,
|
||||
flush_transcript_tail_on_session_end: None,
|
||||
codex_response_item_prefix: None,
|
||||
codex_response_handoff_mode: None,
|
||||
codex_response_handoff_channel_prefixes: None,
|
||||
codex_responses_as_items: None,
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
include_startup_context: None,
|
||||
initial_items: None,
|
||||
realtime_start_instructions: None,
|
||||
realtime_end_instructions: None,
|
||||
prompt: None,
|
||||
realtime_session_id: realtime_session_id.map(str::to_string),
|
||||
transport: Some(ThreadRealtimeStartTransport::ExistingCall {
|
||||
call_id: "rtc_existing".to_string(),
|
||||
}),
|
||||
version,
|
||||
voice: None,
|
||||
})
|
||||
.await?;
|
||||
let _: ThreadRealtimeStartResponse =
|
||||
timeout(DEFAULT_TIMEOUT, harness.mcp.read_response(request_id)).await??;
|
||||
if expected_version == RealtimeConversationVersion::V2 {
|
||||
let error = harness
|
||||
.read_notification::<ThreadRealtimeErrorNotification>("thread/realtime/error")
|
||||
.await?;
|
||||
assert_eq!(
|
||||
error.message,
|
||||
"AVAS realtime calls require realtime v1 or v3"
|
||||
);
|
||||
assert!(harness.realtime_server.handshakes().is_empty());
|
||||
harness.shutdown().await;
|
||||
return Ok(());
|
||||
}
|
||||
let started = harness
|
||||
.read_notification::<ThreadRealtimeStartedNotification>("thread/realtime/started")
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
started,
|
||||
ThreadRealtimeStartedNotification {
|
||||
thread_id: harness.thread_id.clone(),
|
||||
realtime_session_id: realtime_session_id.map(str::to_string),
|
||||
version: expected_version,
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
harness.realtime_server.single_handshake().uri(),
|
||||
expected_handshake_uri
|
||||
);
|
||||
assert!(
|
||||
harness.realtime_server.single_connection().is_empty(),
|
||||
"attaching to an existing call must not overwrite the client session"
|
||||
);
|
||||
assert!(
|
||||
harness
|
||||
.call_capture
|
||||
.requests
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.is_empty(),
|
||||
"an existing call must not issue a second call-create request"
|
||||
);
|
||||
|
||||
harness.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test_case("includeStartupContext"; "rejects startup context")]
|
||||
#[test_case("prompt"; "rejects prompt")]
|
||||
#[test_case("initialItems"; "rejects initial items")]
|
||||
#[test_case("model"; "rejects model")]
|
||||
#[test_case("voice"; "rejects voice")]
|
||||
#[test_case("delegationAckFiller"; "rejects delegation acknowledgement filler")]
|
||||
#[tokio::test]
|
||||
async fn existing_call_rejects_client_owned_session_configuration(option: &str) -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let mut harness = RealtimeE2eHarness::new(
|
||||
RealtimeTestVersion::V1,
|
||||
no_main_loop_responses(),
|
||||
realtime_sideband(Vec::new()),
|
||||
)
|
||||
.await?;
|
||||
let mut params = ThreadRealtimeStartParams {
|
||||
thread_id: harness.thread_id.clone(),
|
||||
client_managed_handoffs: None,
|
||||
delegation_ack_filler: None,
|
||||
flush_transcript_tail_on_session_end: None,
|
||||
codex_response_item_prefix: None,
|
||||
codex_response_handoff_mode: None,
|
||||
codex_response_handoff_channel_prefixes: None,
|
||||
codex_responses_as_items: None,
|
||||
model: None,
|
||||
output_modality: RealtimeOutputModality::Audio,
|
||||
include_startup_context: None,
|
||||
initial_items: None,
|
||||
realtime_start_instructions: None,
|
||||
realtime_end_instructions: None,
|
||||
prompt: None,
|
||||
realtime_session_id: None,
|
||||
transport: Some(ThreadRealtimeStartTransport::ExistingCall {
|
||||
call_id: "rtc_existing".to_string(),
|
||||
}),
|
||||
version: Some(RealtimeConversationVersion::V3),
|
||||
voice: None,
|
||||
};
|
||||
match option {
|
||||
"includeStartupContext" => params.include_startup_context = Some(true),
|
||||
"prompt" => params.prompt = Some(Some("backend prompt".to_string())),
|
||||
"initialItems" => {
|
||||
params.initial_items = Some(vec![ThreadRealtimeInitialItem {
|
||||
role: ConversationTextRole::User,
|
||||
text: "client-owned history".to_string(),
|
||||
}]);
|
||||
}
|
||||
"model" => params.model = Some("another-model".to_string()),
|
||||
"voice" => params.voice = Some(RealtimeVoice::Cove),
|
||||
"delegationAckFiller" => params.delegation_ack_filler = Some(true),
|
||||
option => anyhow::bail!("unsupported test option: {option}"),
|
||||
}
|
||||
|
||||
let request_id = harness
|
||||
.mcp
|
||||
.send_thread_realtime_start_request(params)
|
||||
.await?;
|
||||
let error = timeout(
|
||||
DEFAULT_TIMEOUT,
|
||||
harness
|
||||
.mcp
|
||||
.read_stream_until_error_message(RequestId::Integer(request_id)),
|
||||
)
|
||||
.await??;
|
||||
assert_invalid_request(
|
||||
error,
|
||||
format!("existingCall transport does not support {option}"),
|
||||
);
|
||||
assert!(harness.realtime_server.handshakes().is_empty());
|
||||
|
||||
harness.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn webrtc_v3_start_posts_live_session_and_joins_without_session_update() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
Reference in New Issue
Block a user