diff --git a/codex-rs/app-server-protocol/src/protocol/common.rs b/codex-rs/app-server-protocol/src/protocol/common.rs index 5526f5c548..5936895d9e 100644 --- a/codex-rs/app-server-protocol/src/protocol/common.rs +++ b/codex-rs/app-server-protocol/src/protocol/common.rs @@ -3011,6 +3011,8 @@ mod tests { model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("You are on a call".to_string())), + realtime_start_instructions: Some("Realtime started".to_string()), + realtime_end_instructions: Some("Realtime ended".to_string()), realtime_session_id: Some("sess_456".to_string()), transport: None, version: Some(RealtimeConversationVersion::V1), @@ -3026,6 +3028,8 @@ mod tests { "model": "realtime-treatment-model", "outputModality": "audio", "prompt": "You are on a call", + "realtimeStartInstructions": "Realtime started", + "realtimeEndInstructions": "Realtime ended", "realtimeSessionId": "sess_456", "transport": null, "version": "v1", @@ -3046,6 +3050,8 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3060,6 +3066,8 @@ mod tests { "threadId": "thr_123", "model": null, "outputModality": "audio", + "realtimeStartInstructions": null, + "realtimeEndInstructions": null, "realtimeSessionId": null, "transport": null, "version": null, @@ -3076,6 +3084,8 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(None), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3091,6 +3101,8 @@ mod tests { "model": null, "outputModality": "audio", "prompt": null, + "realtimeStartInstructions": null, + "realtimeEndInstructions": null, "realtimeSessionId": null, "transport": null, "version": null, @@ -3106,6 +3118,8 @@ mod tests { "params": { "threadId": "thr_123", "outputModality": "audio", + "realtimeStartInstructions": null, + "realtimeEndInstructions": null, "realtimeSessionId": null, "transport": null, "voice": null @@ -3123,6 +3137,8 @@ mod tests { "threadId": "thr_123", "outputModality": "audio", "prompt": null, + "realtimeStartInstructions": null, + "realtimeEndInstructions": null, "realtimeSessionId": null, "transport": null, "voice": null @@ -3249,6 +3265,8 @@ mod tests { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("You are on a call".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index a86cd8067d..d89e93f663 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -80,6 +80,10 @@ pub struct ThreadRealtimeStartParams { #[ts(optional = nullable)] pub prompt: Option>, #[ts(optional = nullable)] + pub realtime_start_instructions: Option, + #[ts(optional = nullable)] + pub realtime_end_instructions: Option, + #[ts(optional = nullable)] pub realtime_session_id: Option, #[ts(optional = nullable)] pub transport: Option, diff --git a/codex-rs/app-server/README.md b/codex-rs/app-server/README.md index 42ac510a8a..b5b0360728 100644 --- a/codex-rs/app-server/README.md +++ b/codex-rs/app-server/README.md @@ -831,13 +831,15 @@ const offer = await pc.createOffer(); await pc.setLocalDescription(offer); ``` -Then send `offer.sdp` to app-server. Core uses `experimental_realtime_ws_backend_prompt` for the backend instructions and the thread conversation id as the default Realtime API session identifier. This `realtimeSessionId` value refers to the upstream Realtime API session, not a Codex session/thread-group id. The start response is `{}`; the remote answer SDP arrives later as `thread/realtime/sdp` and should be passed to `setRemoteDescription()`: +Then send `offer.sdp` to app-server. `prompt`, `realtimeStartInstructions`, and `realtimeEndInstructions` override the built-in realtime backend, start, and end prompts for this session. The local `experimental_realtime_ws_backend_prompt`, `experimental_realtime_start_instructions`, and `experimental_realtime_end_instructions` config values take precedence when set. Core uses the thread conversation id as the default Realtime API session identifier. This `realtimeSessionId` value refers to the upstream Realtime API session, not a Codex session/thread-group id. The start response is `{}`; the remote answer SDP arrives later as `thread/realtime/sdp` and should be passed to `setRemoteDescription()`: ```json { "method": "thread/realtime/start", "id": 40, "params": { "threadId": "thr_123", "outputModality": "audio", "prompt": "You are on a call.", + "realtimeStartInstructions": "Realtime conversation started.", + "realtimeEndInstructions": "Realtime conversation ended.", "realtimeSessionId": null, "transport": { "type": "webrtc", "sdp": "v=0\r\no=..." } } } diff --git a/codex-rs/app-server/src/request_processors/turn_processor.rs b/codex-rs/app-server/src/request_processors/turn_processor.rs index b2e6ddc0b9..bc63ef50a9 100644 --- a/codex-rs/app-server/src/request_processors/turn_processor.rs +++ b/codex-rs/app-server/src/request_processors/turn_processor.rs @@ -937,6 +937,8 @@ impl TurnRequestProcessor { model: params.model, output_modality: params.output_modality, prompt: params.prompt, + realtime_start_instructions: params.realtime_start_instructions, + realtime_end_instructions: params.realtime_end_instructions, realtime_session_id: params.realtime_session_id, transport: params.transport.map(|transport| match transport { ThreadRealtimeStartTransport::Websocket => { diff --git a/codex-rs/app-server/tests/suite/v2/experimental_api.rs b/codex-rs/app-server/tests/suite/v2/experimental_api.rs index 03c9d455f0..127a84e8a7 100644 --- a/codex-rs/app-server/tests/suite/v2/experimental_api.rs +++ b/codex-rs/app-server/tests/suite/v2/experimental_api.rs @@ -83,6 +83,8 @@ async fn realtime_conversation_start_requires_experimental_api_capability() -> R model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("hello".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -191,6 +193,8 @@ async fn realtime_webrtc_start_requires_experimental_api_capability() -> Result< model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("hello".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), diff --git a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs index ddc0eae404..c1efe8f1e9 100644 --- a/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs +++ b/codex-rs/app-server/tests/suite/v2/realtime_conversation.rs @@ -317,6 +317,8 @@ impl RealtimeE2eHarness { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: offer_sdp.to_string(), @@ -564,6 +566,8 @@ async fn realtime_conversation_streams_v2_notifications() -> Result<()> { model: Some("realtime-treatment-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: None, + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -819,6 +823,8 @@ async fn realtime_text_output_modality_requests_text_output_and_final_transcript model: None, output_modality: RealtimeOutputModality::Text, prompt: None, + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -995,6 +1001,8 @@ async fn realtime_conversation_stop_emits_closed_notification() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1094,6 +1102,8 @@ async fn realtime_webrtc_start_emits_sdp_notification() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), @@ -2141,6 +2151,8 @@ async fn realtime_webrtc_start_surfaces_backend_error() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ThreadRealtimeStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), @@ -2202,6 +2214,8 @@ async fn realtime_conversation_requires_feature_flag() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 7c060cadb7..6a428851c2 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -391,6 +391,10 @@ pub struct ConfigToml { /// instructions inserted into developer messages when realtime becomes /// active. pub experimental_realtime_start_instructions: Option, + /// Experimental / do not use. Replaces the built-in realtime end + /// instructions inserted into developer messages when realtime becomes + /// inactive. + pub experimental_realtime_end_instructions: Option, /// Experimental / do not use. When set, app-server fetches thread-scoped /// config from a remote service at this endpoint. diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index 9ed02fdd17..e74153a402 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -4478,6 +4478,10 @@ "experimental_compact_prompt_file": { "$ref": "#/definitions/AbsolutePathBuf" }, + "experimental_realtime_end_instructions": { + "description": "Experimental / do not use. Replaces the built-in realtime end instructions inserted into developer messages when realtime becomes inactive.", + "type": "string" + }, "experimental_realtime_start_instructions": { "description": "Experimental / do not use. Replaces the built-in realtime start instructions inserted into developer messages when realtime becomes active.", "type": "string" diff --git a/codex-rs/core/src/config/config_tests.rs b/codex-rs/core/src/config/config_tests.rs index c19fe76fb5..d6bfa45b37 100644 --- a/codex-rs/core/src/config/config_tests.rs +++ b/codex-rs/core/src/config/config_tests.rs @@ -10514,6 +10514,35 @@ experimental_realtime_start_instructions = "start instructions from config" Ok(()) } +#[tokio::test] +async fn experimental_realtime_end_instructions_load_from_config_toml() -> std::io::Result<()> { + let cfg: ConfigToml = toml::from_str( + r#" +experimental_realtime_end_instructions = "end instructions from config" +"#, + ) + .expect("TOML deserialization should succeed"); + + assert_eq!( + cfg.experimental_realtime_end_instructions.as_deref(), + Some("end instructions from config") + ); + + let codex_home = TempDir::new()?; + let config = Config::load_from_base_config_with_overrides( + cfg, + ConfigOverrides::default(), + codex_home.abs(), + ) + .await?; + + assert_eq!( + config.experimental_realtime_end_instructions.as_deref(), + Some("end instructions from config") + ); + Ok(()) +} + #[tokio::test] async fn experimental_thread_config_endpoint_loads_from_config_toml() -> std::io::Result<()> { let cfg: ConfigToml = toml::from_str( diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index cadb797212..b1811b9892 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -959,6 +959,10 @@ pub struct Config { /// instructions inserted into developer messages when realtime becomes /// active. pub experimental_realtime_start_instructions: Option, + /// Experimental / do not use. Replaces the built-in realtime end + /// instructions inserted into developer messages when realtime becomes + /// inactive. + pub experimental_realtime_end_instructions: Option, /// Experimental / do not use. When set, app-server fetches thread-scoped /// config from a remote service at this endpoint. pub experimental_thread_config_endpoint: Option, @@ -3553,6 +3557,7 @@ impl Config { experimental_realtime_ws_backend_prompt: cfg.experimental_realtime_ws_backend_prompt, experimental_realtime_ws_startup_context: cfg.experimental_realtime_ws_startup_context, experimental_realtime_start_instructions: cfg.experimental_realtime_start_instructions, + experimental_realtime_end_instructions: cfg.experimental_realtime_end_instructions, experimental_thread_config_endpoint: cfg.experimental_thread_config_endpoint, experimental_thread_store: thread_store_config(cfg.experimental_thread_store), forced_chatgpt_workspace_id, diff --git a/codex-rs/core/src/context/realtime_end_instructions.rs b/codex-rs/core/src/context/realtime_end_instructions.rs index b4225dfb49..ee4c2e2867 100644 --- a/codex-rs/core/src/context/realtime_end_instructions.rs +++ b/codex-rs/core/src/context/realtime_end_instructions.rs @@ -5,12 +5,21 @@ use codex_protocol::protocol::REALTIME_CONVERSATION_OPEN_TAG; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct RealtimeEndInstructions { + instructions: String, reason: String, } impl RealtimeEndInstructions { pub(crate) fn new(reason: impl Into) -> Self { + Self::with_instructions(reason, END_INSTRUCTIONS.trim()) + } + + pub(crate) fn with_instructions( + reason: impl Into, + instructions: impl Into, + ) -> Self { Self { + instructions: instructions.into(), reason: reason.into(), } } @@ -33,6 +42,6 @@ impl ContextualUserFragment for RealtimeEndInstructions { } fn body(&self) -> String { - format!("\n{}\n\nReason: {}\n", END_INSTRUCTIONS.trim(), self.reason) + format!("\n{}\n\nReason: {}\n", self.instructions, self.reason) } } diff --git a/codex-rs/core/src/context_manager/updates.rs b/codex-rs/core/src/context_manager/updates.rs index d7302bbbf8..910c7aa67f 100644 --- a/codex-rs/core/src/context_manager/updates.rs +++ b/codex-rs/core/src/context_manager/updates.rs @@ -100,7 +100,7 @@ pub(crate) fn build_realtime_update_item( previous.and_then(|item| item.realtime_active), next.realtime_active, ) { - (Some(true), false) => Some(RealtimeEndInstructions::new("inactive").render()), + (Some(true), false) => Some(render_realtime_end(next)), (Some(false), true) | (None, true) => Some( if let Some(instructions) = next .config @@ -116,7 +116,20 @@ pub(crate) fn build_realtime_update_item( (None, false) => previous_turn_settings .and_then(|settings| settings.realtime_active) .filter(|realtime_active| *realtime_active) - .map(|_| RealtimeEndInstructions::new("inactive").render()), + .map(|_| render_realtime_end(next)), + } +} + +fn render_realtime_end(next: &TurnContext) -> String { + match next + .config + .experimental_realtime_end_instructions + .as_deref() + { + Some(instructions) => { + RealtimeEndInstructions::with_instructions("inactive", instructions).render() + } + None => RealtimeEndInstructions::new("inactive").render(), } } diff --git a/codex-rs/core/src/realtime_conversation.rs b/codex-rs/core/src/realtime_conversation.rs index 33568f6f38..07ab9557d0 100644 --- a/codex-rs/core/src/realtime_conversation.rs +++ b/codex-rs/core/src/realtime_conversation.rs @@ -614,6 +614,8 @@ pub(crate) async fn handle_start( struct PreparedRealtimeConversationStart { api_provider: ApiProvider, extra_headers: Option, + realtime_end_instructions: Option, + realtime_start_instructions: Option, requested_realtime_session_id: Option, version: RealtimeWsVersion, session_config: RealtimeSessionConfig, @@ -671,6 +673,8 @@ async fn prepare_realtime_start( Ok(PreparedRealtimeConversationStart { api_provider, extra_headers, + realtime_end_instructions: params.realtime_end_instructions, + realtime_start_instructions: params.realtime_start_instructions, requested_realtime_session_id, version, session_config, @@ -787,6 +791,8 @@ async fn handle_start_inner( let PreparedRealtimeConversationStart { api_provider, extra_headers, + realtime_end_instructions, + realtime_start_instructions, requested_realtime_session_id, version, session_config, @@ -805,6 +811,8 @@ async fn handle_start_inner( sdp, }; let start_output = sess.conversation.start(start).await?; + sess.set_realtime_instruction_overrides(realtime_start_instructions, realtime_end_instructions) + .await; info!("realtime conversation started"); diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 8424ce216f..01fefab88f 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -601,6 +601,8 @@ impl Codex { personality: config.personality, base_instructions, compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: session_permission_profile_state_from_config(&config)?, @@ -1493,6 +1495,16 @@ impl Session { .clone() } + pub(crate) async fn set_realtime_instruction_overrides( + &self, + realtime_start_instructions: Option, + realtime_end_instructions: Option, + ) { + let mut state = self.state.lock().await; + state.session_configuration.realtime_start_instructions = realtime_start_instructions; + state.session_configuration.realtime_end_instructions = realtime_end_instructions; + } + pub(crate) async fn provider(&self) -> ModelProviderInfo { let state = self.state.lock().await; state.session_configuration.provider.clone() diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index e42cf202e8..e22b19e344 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -68,6 +68,9 @@ pub(crate) struct SessionConfiguration { /// Compact prompt override. pub(super) compact_prompt: Option, + pub(super) realtime_start_instructions: Option, + pub(super) realtime_end_instructions: Option, + /// When to escalate for approval for execution pub(super) approval_policy: Constrained, pub(super) approvals_reviewer: ApprovalsReviewer, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 29de7ca662..f3d2a9f1f5 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -3178,6 +3178,8 @@ async fn set_rate_limits_retains_previous_credits() { .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -3285,6 +3287,8 @@ async fn set_rate_limits_updates_plan_type_when_present() { .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -3817,6 +3821,8 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -4669,6 +4675,8 @@ async fn session_new_fails_when_zsh_fork_enabled_without_packaged_zsh() { .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -4777,6 +4785,8 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -5009,6 +5019,8 @@ async fn make_session_with_config_and_rx( .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -5111,6 +5123,8 @@ async fn make_session_with_history_source_and_agent_control_and_rx( .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -6855,6 +6869,8 @@ where .clone() .unwrap_or_else(|| model_info.get_model_instructions(config.personality)), compact_prompt: config.compact_prompt.clone(), + realtime_start_instructions: None, + realtime_end_instructions: None, approval_policy: config.permissions.approval_policy.clone(), approvals_reviewer: config.approvals_reviewer, permission_profile_state: config.permissions.permission_profile_state().clone(), @@ -7284,6 +7300,80 @@ async fn build_settings_update_items_omits_environment_item_when_disabled() { ); } +#[tokio::test] +async fn realtime_instruction_overrides_are_stored_for_future_turns() { + let (session, _) = make_session_and_context().await; + + session + .set_realtime_instruction_overrides( + Some("request realtime start".to_string()), + Some("request realtime end".to_string()), + ) + .await; + + let state = session.state.lock().await; + assert_eq!( + state.session_configuration.realtime_start_instructions, + Some("request realtime start".to_string()) + ); + assert_eq!( + state.session_configuration.realtime_end_instructions, + Some("request realtime end".to_string()) + ); + + let per_turn_config = Session::build_per_turn_config( + &state.session_configuration, + state.session_configuration.cwd().clone(), + ); + assert_eq!( + per_turn_config + .experimental_realtime_start_instructions + .as_deref(), + Some("request realtime start") + ); + assert_eq!( + per_turn_config + .experimental_realtime_end_instructions + .as_deref(), + Some("request realtime end") + ); +} + +#[tokio::test] +async fn local_realtime_instruction_config_takes_precedence_over_request_overrides() { + let session = make_session_with_config(|config| { + config.experimental_realtime_start_instructions = Some("local realtime start".to_string()); + config.experimental_realtime_end_instructions = Some("local realtime end".to_string()); + }) + .await + .expect("create session"); + + session + .set_realtime_instruction_overrides( + Some("request realtime start".to_string()), + Some("request realtime end".to_string()), + ) + .await; + + let state = session.state.lock().await; + let per_turn_config = Session::build_per_turn_config( + &state.session_configuration, + state.session_configuration.cwd().clone(), + ); + assert_eq!( + per_turn_config + .experimental_realtime_start_instructions + .as_deref(), + Some("local realtime start") + ); + assert_eq!( + per_turn_config + .experimental_realtime_end_instructions + .as_deref(), + Some("local realtime end") + ); +} + #[tokio::test] async fn build_settings_update_items_emits_realtime_start_when_session_becomes_live() { let (session, previous_context) = make_session_and_context().await; @@ -7294,6 +7384,8 @@ async fn build_settings_update_items_emits_realtime_start_when_session_becomes_l &session.services.models_manager, ) .await; + Arc::make_mut(&mut current_context.config).experimental_realtime_start_instructions = + Some("custom realtime start instructions".to_string()); current_context.realtime_active = true; let update_items = session @@ -7307,7 +7399,7 @@ async fn build_settings_update_items_emits_realtime_start_when_session_becomes_l assert!( developer_texts .iter() - .any(|text| text.contains("")), + .any(|text| text.contains("custom realtime start instructions")), "expected a realtime start update, got {developer_texts:?}" ); } @@ -7322,6 +7414,8 @@ async fn build_settings_update_items_emits_realtime_end_when_session_stops_being &session.services.models_manager, ) .await; + Arc::make_mut(&mut current_context.config).experimental_realtime_end_instructions = + Some("custom realtime end instructions".to_string()); current_context.realtime_active = false; let update_items = session @@ -7335,7 +7429,7 @@ async fn build_settings_update_items_emits_realtime_end_when_session_stops_being assert!( developer_texts .iter() - .any(|text| text.contains("Reason: inactive")), + .any(|text| text.contains("custom realtime end instructions")), "expected a realtime end update, got {developer_texts:?}" ); } diff --git a/codex-rs/core/src/session/turn_context.rs b/codex-rs/core/src/session/turn_context.rs index 10eff3f3b4..b4a74d1223 100644 --- a/codex-rs/core/src/session/turn_context.rs +++ b/codex-rs/core/src/session/turn_context.rs @@ -405,6 +405,20 @@ impl Session { // todo(aibrahim): store this state somewhere else so we don't need to mut config let config = session_configuration.original_config_do_not_use.clone(); let mut per_turn_config = (*config).clone(); + if per_turn_config + .experimental_realtime_start_instructions + .is_none() + { + per_turn_config.experimental_realtime_start_instructions = + session_configuration.realtime_start_instructions.clone(); + } + if per_turn_config + .experimental_realtime_end_instructions + .is_none() + { + per_turn_config.experimental_realtime_end_instructions = + session_configuration.realtime_end_instructions.clone(); + } per_turn_config.cwd = cwd; per_turn_config.workspace_roots = session_configuration.workspace_roots.clone(); per_turn_config diff --git a/codex-rs/core/tests/suite/compact_remote.rs b/codex-rs/core/tests/suite/compact_remote.rs index eee445ea9f..6db98f527b 100644 --- a/codex-rs/core/tests/suite/compact_remote.rs +++ b/codex-rs/core/tests/suite/compact_remote.rs @@ -197,6 +197,8 @@ async fn start_realtime_conversation(codex: &codex_core::CodexThread) -> Result< model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, diff --git a/codex-rs/core/tests/suite/realtime_conversation.rs b/codex-rs/core/tests/suite/realtime_conversation.rs index 25f0516b0d..e2616fe3d3 100644 --- a/codex-rs/core/tests/suite/realtime_conversation.rs +++ b/codex-rs/core/tests/suite/realtime_conversation.rs @@ -285,6 +285,8 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -425,6 +427,8 @@ async fn conversation_start_defaults_to_v2_and_gpt_realtime_1_5() -> Result<()> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -515,6 +519,8 @@ async fn conversation_webrtc_start_posts_generated_session() -> Result<()> { model: Some("session-override-model".to_string()), output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), @@ -687,6 +693,8 @@ async fn conversation_webrtc_close_while_sideband_connecting_drops_pending_join( model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), @@ -773,6 +781,8 @@ async fn conversation_webrtc_sideband_connect_failure_closes_with_error() -> Res model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: Some(ConversationStartTransport::Webrtc { sdp: "v=offer\r\n".to_string(), @@ -860,6 +870,8 @@ async fn conversation_start_uses_openai_env_key_fallback_with_chatgpt_auth() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -928,6 +940,8 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1020,6 +1034,8 @@ async fn conversation_start_preflight_failure_emits_realtime_error_only() -> Res model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1066,6 +1082,8 @@ async fn conversation_start_connect_failure_emits_realtime_error_only() -> Resul model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1159,6 +1177,8 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("old".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: Some("conv_old".to_string()), transport: None, version: None, @@ -1184,6 +1204,8 @@ async fn conversation_second_start_replaces_runtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("new".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: Some("conv_new".to_string()), transport: None, version: None, @@ -1280,6 +1302,8 @@ async fn conversation_uses_experimental_realtime_ws_base_url_override() -> Resul model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1344,6 +1368,8 @@ async fn conversation_uses_default_realtime_backend_prompt() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: None, + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1416,6 +1442,8 @@ async fn conversation_uses_empty_instructions_for_null_or_empty_prompt() -> Resu model: None, output_modality: RealtimeOutputModality::Audio, prompt, + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1481,6 +1509,8 @@ async fn conversation_uses_explicit_start_voice() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1538,6 +1568,8 @@ async fn conversation_uses_configured_realtime_voice() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1583,6 +1615,8 @@ async fn conversation_rejects_voice_for_wrong_realtime_version() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1629,6 +1663,8 @@ async fn conversation_uses_experimental_realtime_ws_backend_prompt_override() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1701,6 +1737,8 @@ async fn conversation_uses_experimental_realtime_ws_startup_context_override() - model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1767,6 +1805,8 @@ async fn conversation_disables_realtime_startup_context_with_empty_override() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("prompt from op".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1826,6 +1866,8 @@ async fn conversation_start_injects_startup_context_from_thread_history() -> Res model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -1937,6 +1979,8 @@ async fn conversation_startup_context_current_thread_selects_many_turns_by_budge model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2044,6 +2088,8 @@ async fn conversation_startup_context_falls_back_to_workspace_map() -> Result<() model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2103,6 +2149,8 @@ async fn conversation_startup_context_is_truncated_and_sent_once_per_start() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2182,6 +2230,8 @@ async fn conversation_user_text_turn_is_not_sent_to_realtime() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2278,6 +2328,8 @@ async fn realtime_v2_noop_tool_call_returns_empty_function_output_without_respon model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2376,6 +2428,8 @@ async fn conversation_mirrors_assistant_message_text_to_realtime_handoff() -> Re model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2512,6 +2566,8 @@ async fn conversation_handoff_persists_across_item_done_until_turn_complete() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2663,6 +2719,8 @@ async fn inbound_handoff_request_starts_turn() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2764,6 +2822,8 @@ async fn inbound_handoff_request_uses_active_transcript() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2866,6 +2926,8 @@ async fn inbound_handoff_request_sends_transcript_delta_after_each_handoff() -> model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -2966,6 +3028,8 @@ async fn inbound_conversation_item_does_not_start_turn_and_still_forwards_audio( model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3088,6 +3152,8 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3240,6 +3306,8 @@ async fn inbound_handoff_request_does_not_block_realtime_event_forwarding() -> R model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3376,6 +3444,8 @@ async fn inbound_handoff_request_steers_active_turn() -> Result<()> { model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, @@ -3528,6 +3598,8 @@ async fn inbound_handoff_request_starts_turn_and_does_not_block_realtime_audio() model: None, output_modality: RealtimeOutputModality::Audio, prompt: Some(Some("backend prompt".to_string())), + realtime_start_instructions: None, + realtime_end_instructions: None, realtime_session_id: None, transport: None, version: None, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 01e664c734..6e639a5aba 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -188,6 +188,8 @@ pub struct ConversationStartParams { /// Selects whether the realtime session should produce text or audio output. pub output_modality: RealtimeOutputModality, pub prompt: Option>, + pub realtime_start_instructions: Option, + pub realtime_end_instructions: Option, pub realtime_session_id: Option, pub transport: Option, /// Overrides the configured realtime protocol version for this session only.