Use transient developer instructions for realtime sessions

This commit is contained in:
Ahmed Ibrahim
2026-02-27 11:35:25 -08:00
parent be2529b0eb
commit 8d4dcadd7c
16 changed files with 254 additions and 178 deletions
@@ -885,7 +885,6 @@
"description": "Initial collaboration mode to use when the TUI starts.",
"enum": [
"plan",
"realtime",
"default"
],
"type": "string"
@@ -3817,7 +3817,6 @@
"description": "Initial collaboration mode to use when the TUI starts.",
"enum": [
"plan",
"realtime",
"default"
],
"type": "string"
@@ -10219,7 +10219,6 @@
"description": "Initial collaboration mode to use when the TUI starts.",
"enum": [
"plan",
"realtime",
"default"
],
"type": "string"
@@ -86,7 +86,6 @@
"description": "Initial collaboration mode to use when the TUI starts.",
"enum": [
"plan",
"realtime",
"default"
],
"type": "string"
@@ -5,4 +5,4 @@
/**
* Initial collaboration mode to use when the TUI starts.
*/
export type ModeKind = "plan" | "realtime" | "default";
export type ModeKind = "plan" | "default";
+93 -47
View File
@@ -162,6 +162,10 @@ pub enum SteerInputError {
ExpectedTurnMismatch { expected: String, actual: String },
EmptyInput,
}
const REALTIME_START_DEVELOPER_INSTRUCTIONS: &str = "Realtime conversation handling is now active. Keep responses concise, low-latency, interruption-safe, and operationally safe for live interaction. This instruction applies only while realtime is active.";
const REALTIME_END_DEVELOPER_INSTRUCTIONS: &str = "Realtime conversation handling has ended. The realtime-specific developer instruction no longer applies.";
use crate::exec_policy::ExecPolicyUpdateError;
use crate::feedback_tags;
use crate::file_watcher::FileWatcher;
@@ -3035,45 +3039,69 @@ impl Session {
state.session_configuration.collaboration_mode.clone()
}
pub(crate) async fn enter_realtime_collaboration_mode(&self) -> ConstraintResult<()> {
let current = self.collaboration_mode().await;
let collaboration_mode = self
.services
.models_manager
.list_collaboration_modes()
.into_iter()
.find(|mask| mask.mode == Some(ModeKind::Realtime))
.map(|mask| current.apply_mask(&mask))
.unwrap_or_else(|| CollaborationMode {
mode: ModeKind::Realtime,
settings: Settings {
model: current.model().to_string(),
reasoning_effort: current.reasoning_effort(),
developer_instructions: None,
},
});
self.update_settings(SessionSettingsUpdate {
collaboration_mode: Some(collaboration_mode),
..Default::default()
})
.await
pub(crate) async fn realtime_instructions_active(&self) -> bool {
let state = self.state.lock().await;
state.realtime_instructions_active()
}
pub(crate) async fn reset_to_default_collaboration_mode(&self) -> ConstraintResult<()> {
let current = self.collaboration_mode().await;
self.update_settings(SessionSettingsUpdate {
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: current.model().to_string(),
reasoning_effort: current.reasoning_effort(),
developer_instructions: None,
},
}),
..Default::default()
})
.await
pub(crate) async fn emit_realtime_start_instructions(&self) {
if self.realtime_instructions_active().await {
return;
}
self.record_session_developer_message(REALTIME_START_DEVELOPER_INSTRUCTIONS)
.await;
let mut state = self.state.lock().await;
state.set_realtime_instructions_active(true);
}
pub(crate) async fn emit_realtime_end_instructions(&self) {
if !self.realtime_instructions_active().await {
return;
}
self.record_session_developer_message(REALTIME_END_DEVELOPER_INSTRUCTIONS)
.await;
let mut state = self.state.lock().await;
state.set_realtime_instructions_active(false);
}
async fn record_session_developer_message(&self, text: &str) {
let message: ResponseItem = DeveloperInstructions::new(text.to_string()).into();
let active_turn_context = {
let active = self.active_turn.lock().await;
active
.as_ref()
.and_then(|active_turn| active_turn.tasks.first())
.map(|(_, task)| Arc::clone(&task.turn_context))
};
if let Some(turn_context) = active_turn_context {
self.record_conversation_items(turn_context.as_ref(), std::slice::from_ref(&message))
.await;
return;
}
let session_configuration = {
let state = self.state.lock().await;
state.session_configuration.clone()
};
let per_turn_config = Self::build_per_turn_config(&session_configuration);
let model_info = self
.services
.models_manager
.get_model_info(
session_configuration.collaboration_mode.model(),
&per_turn_config,
)
.await;
{
let mut state = self.state.lock().await;
state.record_items(
std::slice::from_ref(&message).iter(),
model_info.truncation_policy.into(),
);
}
self.persist_rollout_response_items(std::slice::from_ref(&message))
.await;
}
async fn send_raw_response_items(&self, turn_context: &TurnContext, items: &[ResponseItem]) {
@@ -4592,9 +4620,7 @@ mod handlers {
pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
sess.abort_all_tasks(TurnAbortReason::Interrupted).await;
let _ = sess.conversation.shutdown().await;
if let Err(err) = sess.reset_to_default_collaboration_mode().await {
warn!("failed to reset collaboration mode during shutdown: {err}");
}
sess.emit_realtime_end_instructions().await;
sess.services
.unified_exec_manager
.terminate_all_processes()
@@ -8967,16 +8993,36 @@ mod tests {
}
#[tokio::test]
async fn shutdown_resets_realtime_collaboration_mode_to_default() {
async fn shutdown_emits_realtime_end_instructions() {
let (session, _turn_context, _rx) = make_session_and_context_with_rx().await;
session
.enter_realtime_collaboration_mode()
.await
.expect("enter realtime collaboration mode");
assert_eq!(session.collaboration_mode().await.mode, ModeKind::Realtime);
session.emit_realtime_start_instructions().await;
assert!(session.realtime_instructions_active().await);
assert!(handlers::shutdown(&session, "shutdown-test".to_string()).await);
assert_eq!(session.collaboration_mode().await.mode, ModeKind::Default);
assert!(!session.realtime_instructions_active().await);
let developer_texts = session
.clone_history()
.await
.raw_items()
.iter()
.filter_map(|item| match item {
ResponseItem::Message { role, content, .. } if role == "developer" => {
content.iter().find_map(|content| match content {
ContentItem::InputText { text } => Some(text.clone()),
_ => None,
})
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(
developer_texts,
vec![
REALTIME_START_DEVELOPER_INSTRUCTIONS.to_string(),
REALTIME_END_DEVELOPER_INSTRUCTIONS.to_string(),
]
);
}
#[tokio::test]
@@ -6,8 +6,6 @@ use codex_protocol::openai_models::ReasoningEffort;
const COLLABORATION_MODE_PLAN: &str = include_str!("../../templates/collaboration_mode/plan.md");
const COLLABORATION_MODE_DEFAULT: &str =
include_str!("../../templates/collaboration_mode/default.md");
const COLLABORATION_MODE_REALTIME: &str =
include_str!("../../templates/collaboration_mode/realtime.md");
const KNOWN_MODE_NAMES_PLACEHOLDER: &str = "{{KNOWN_MODE_NAMES}}";
const REQUEST_USER_INPUT_AVAILABILITY_PLACEHOLDER: &str = "{{REQUEST_USER_INPUT_AVAILABILITY}}";
const ASKING_QUESTIONS_GUIDANCE_PLACEHOLDER: &str = "{{ASKING_QUESTIONS_GUIDANCE}}";
@@ -26,11 +24,7 @@ pub struct CollaborationModesConfig {
pub(crate) fn builtin_collaboration_mode_presets(
collaboration_modes_config: CollaborationModesConfig,
) -> Vec<CollaborationModeMask> {
vec![
plan_preset(),
default_preset(collaboration_modes_config),
realtime_preset(),
]
vec![plan_preset(), default_preset(collaboration_modes_config)]
}
fn plan_preset() -> CollaborationModeMask {
@@ -53,16 +47,6 @@ fn default_preset(collaboration_modes_config: CollaborationModesConfig) -> Colla
}
}
fn realtime_preset() -> CollaborationModeMask {
CollaborationModeMask {
name: ModeKind::Realtime.display_name().to_string(),
mode: Some(ModeKind::Realtime),
model: None,
reasoning_effort: None,
developer_instructions: Some(Some(COLLABORATION_MODE_REALTIME.to_string())),
}
}
fn default_mode_instructions(collaboration_modes_config: CollaborationModesConfig) -> String {
let known_mode_names = format_mode_names(&TUI_VISIBLE_COLLABORATION_MODES);
let request_user_input_availability = request_user_input_availability_message(
@@ -134,8 +118,6 @@ mod tests {
plan_preset().reasoning_effort,
Some(Some(ReasoningEffort::Medium))
);
assert_eq!(realtime_preset().name, ModeKind::Realtime.display_name());
assert_eq!(realtime_preset().reasoning_effort, None);
}
#[test]
@@ -174,19 +156,4 @@ mod tests {
.contains("ask the user directly with a concise plain-text question")
);
}
#[test]
fn realtime_preset_only_sets_developer_instructions() {
let realtime = realtime_preset();
assert_eq!(realtime.mode, Some(ModeKind::Realtime));
assert_eq!(realtime.model, None);
assert_eq!(realtime.reasoning_effort, None);
assert!(
realtime
.developer_instructions
.expect("realtime preset should include instructions")
.expect("realtime instructions should be set")
.contains("Realtime mode")
);
}
}
+8 -16
View File
@@ -186,6 +186,9 @@ pub(crate) async fn handle_start(
.session_id
.or_else(|| Some(sess.conversation_id.to_string()));
info!("starting realtime conversation");
if sess.realtime_instructions_active().await {
sess.emit_realtime_end_instructions().await;
}
let events_rx = match sess
.conversation
.start(api_provider, None, prompt, requested_session_id.clone())
@@ -198,11 +201,7 @@ pub(crate) async fn handle_start(
return Ok(());
}
};
if let Err(err) = sess.enter_realtime_collaboration_mode().await {
let _ = sess.conversation.shutdown().await;
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest).await;
return Ok(());
}
sess.emit_realtime_start_instructions().await;
info!("realtime conversation started");
@@ -243,7 +242,7 @@ pub(crate) async fn handle_start(
}
if let Some(()) = sess_clone.conversation.running_state().await {
info!("realtime conversation transport closed");
close_realtime_and_reset_mode(
close_realtime_and_emit_end_instruction(
&sess_clone,
ev,
"transport_closed",
@@ -304,7 +303,7 @@ pub(crate) async fn handle_text(
pub(crate) async fn handle_close(sess: &Arc<Session>, sub_id: String) {
let error_sub_id = sub_id.clone();
close_realtime_and_reset_mode(
close_realtime_and_emit_end_instruction(
sess,
|msg| Event {
id: sub_id.clone(),
@@ -316,7 +315,7 @@ pub(crate) async fn handle_close(sess: &Arc<Session>, sub_id: String) {
.await;
}
async fn close_realtime_and_reset_mode<F>(
async fn close_realtime_and_emit_end_instruction<F>(
sess: &Arc<Session>,
event_builder: F,
reason: &str,
@@ -333,14 +332,7 @@ async fn close_realtime_and_reset_mode<F>(
return;
}
if let Err(err) = sess.reset_to_default_collaboration_mode().await {
if let Some(sub_id) = error_sub_id {
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::Other).await;
} else {
warn!("failed to reset collaboration mode after realtime close: {err}");
}
return;
}
sess.emit_realtime_end_instructions().await;
sess.send_event_raw(event_builder(EventMsg::RealtimeConversationClosed(
RealtimeConversationClosedEvent {
+10
View File
@@ -19,6 +19,7 @@ use codex_protocol::protocol::TurnContextItem;
pub(crate) struct SessionState {
pub(crate) session_configuration: SessionConfiguration,
pub(crate) history: ContextManager,
pub(crate) realtime_instructions_active: bool,
pub(crate) latest_rate_limits: Option<RateLimitSnapshot>,
pub(crate) server_reasoning_included: bool,
pub(crate) dependency_env: HashMap<String, String>,
@@ -40,6 +41,7 @@ impl SessionState {
Self {
session_configuration,
history,
realtime_instructions_active: false,
latest_rate_limits: None,
server_reasoning_included: false,
dependency_env: HashMap::new(),
@@ -71,6 +73,14 @@ impl SessionState {
self.history.clone()
}
pub(crate) fn realtime_instructions_active(&self) -> bool {
self.realtime_instructions_active
}
pub(crate) fn set_realtime_instructions_active(&mut self, active: bool) {
self.realtime_instructions_active = active;
}
pub(crate) fn replace_history(
&mut self,
items: Vec<ResponseItem>,
@@ -132,7 +132,6 @@ mod tests {
fn request_user_input_mode_availability_defaults_to_plan_only() {
assert!(ModeKind::Plan.allows_request_user_input());
assert!(!ModeKind::Default.allows_request_user_input());
assert!(!ModeKind::Realtime.allows_request_user_input());
assert!(!ModeKind::Execute.allows_request_user_input());
assert!(!ModeKind::PairProgramming.allows_request_user_input());
}
@@ -151,10 +150,6 @@ mod tests {
request_user_input_unavailable_message(ModeKind::Default, true),
None
);
assert_eq!(
request_user_input_unavailable_message(ModeKind::Realtime, false),
Some("request_user_input is unavailable in Realtime mode".to_string())
);
assert_eq!(
request_user_input_unavailable_message(ModeKind::Execute, false),
Some("request_user_input is unavailable in Execute mode".to_string())
@@ -1,13 +0,0 @@
# Collaboration Mode: Realtime
You are now in Realtime mode. Any previous instructions for other collaboration modes are no longer active.
Realtime mode is active only while handling an active realtime conversation session.
- Keep responses concise and low-latency.
- Prefer interruption-safe phrasing that works well for live streaming and audio playback.
- Avoid unnecessary preamble or long digressions while realtime is active.
Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it.
When the realtime conversation closes, the session returns to Default mode.
@@ -561,7 +561,7 @@ async fn collaboration_mode_update_emits_empty_marker_when_instructions_clear()
.await;
let test = test_codex().build(&server).await?;
let realtime_text = "realtime mode instructions";
let plan_text = "plan mode instructions";
test.codex
.submit(Op::OverrideTurnContext {
@@ -573,8 +573,8 @@ async fn collaboration_mode_update_emits_empty_marker_when_instructions_clear()
effort: None,
summary: None,
collaboration_mode: Some(collab_mode_with_mode_and_instructions(
ModeKind::Realtime,
Some(realtime_text),
ModeKind::Plan,
Some(plan_text),
)),
personality: None,
})
@@ -621,7 +621,7 @@ async fn collaboration_mode_update_emits_empty_marker_when_instructions_clear()
let input = req2.single_request().input();
let dev_texts = developer_texts(&input);
assert_eq!(count_exact(&dev_texts, &collab_xml(realtime_text)), 1);
assert_eq!(count_exact(&dev_texts, &collab_xml(plan_text)), 1);
Ok(())
}
@@ -25,6 +25,25 @@ use serde_json::json;
use std::time::Duration;
use tokio::sync::oneshot;
fn developer_texts(input: &[Value]) -> Vec<String> {
input
.iter()
.filter_map(|item| {
let role = item.get("role")?.as_str()?;
if role != "developer" {
return None;
}
let text = item
.get("content")?
.as_array()?
.first()?
.get("text")?
.as_str()?;
Some(text.to_string())
})
.collect()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -160,6 +179,61 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_start_persists_realtime_start_instruction_into_next_turn() -> Result<()> {
skip_if_no_network!(Ok(()));
let api_server = start_mock_server().await;
let response_mock = responses::mount_sse_once(
&api_server,
responses::sse(vec![
responses::ev_response_created("resp_1"),
responses::ev_completed("resp_1"),
]),
)
.await;
let realtime_server = start_websocket_server(vec![vec![vec![json!({
"type": "session.created",
"session": { "id": "sess_1" }
})]]])
.await;
let mut builder = test_codex().with_config({
let realtime_base_url = realtime_server.uri().to_string();
move |config| {
config.experimental_realtime_ws_base_url = Some(realtime_base_url);
}
});
let test = builder.build(&api_server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
}))
.await?;
let _session_created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::SessionCreated { session_id },
}) => Some(session_id.clone()),
_ => None,
})
.await;
test.submit_turn("hello").await?;
let dev_texts = developer_texts(&response_mock.single_request().input());
assert!(dev_texts.iter().any(|text| {
text.contains("Realtime conversation handling is now active.")
&& text.contains("This instruction applies only while realtime is active.")
}));
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_transport_close_emits_closed_event() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -208,6 +282,68 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_close_persists_realtime_end_instruction_into_next_turn() -> Result<()> {
skip_if_no_network!(Ok(()));
let api_server = start_mock_server().await;
let response_mock = responses::mount_sse_once(
&api_server,
responses::sse(vec![
responses::ev_response_created("resp_1"),
responses::ev_completed("resp_1"),
]),
)
.await;
let realtime_server = start_websocket_server(vec![vec![vec![json!({
"type": "session.created",
"session": { "id": "sess_1" }
})]]])
.await;
let mut builder = test_codex().with_config({
let realtime_base_url = realtime_server.uri().to_string();
move |config| {
config.experimental_realtime_ws_base_url = Some(realtime_base_url);
}
});
let test = builder.build(&api_server).await?;
test.codex
.submit(Op::RealtimeConversationStart(ConversationStartParams {
prompt: "backend prompt".to_string(),
session_id: None,
}))
.await?;
let _session_created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::SessionCreated { session_id },
}) => Some(session_id.clone()),
_ => None,
})
.await;
test.codex.submit(Op::RealtimeConversationClose).await?;
let _closed = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationClosed(closed) => Some(closed.clone()),
_ => None,
})
.await;
test.submit_turn("hello").await?;
let dev_texts = developer_texts(&response_mock.single_request().input());
assert!(dev_texts.iter().any(|text| {
text.contains("Realtime conversation handling has ended.")
&& text.contains("realtime-specific developer instruction no longer applies")
}));
realtime_server.shutdown().await;
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn conversation_audio_before_start_emits_error() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -711,7 +847,6 @@ async fn inbound_realtime_text_ignores_user_role_and_still_forwards_audio() -> R
async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_audio() -> Result<()>
{
skip_if_no_network!(Ok(()));
let start = std::time::Instant::now();
let (gate_completed_tx, gate_completed_rx) = oneshot::channel();
let first_chunks = vec![
@@ -803,22 +938,9 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
_ => None,
})
.await;
eprintln!(
"[realtime test +{}ms] saw trigger text={:?}",
start.elapsed().as_millis(),
"delegate now"
);
let mirrored_request = realtime_server.wait_for_request(0, 1).await;
let mirrored_request_body = mirrored_request.body_json();
eprintln!(
"[realtime test +{}ms] saw mirrored request type={:?} role={:?} text={:?} data={:?}",
start.elapsed().as_millis(),
mirrored_request_body["type"].as_str(),
mirrored_request_body["item"]["role"].as_str(),
mirrored_request_body["item"]["content"][0]["text"].as_str(),
mirrored_request_body["item"]["content"][0]["data"].as_str(),
);
assert_eq!(
mirrored_request_body["type"].as_str(),
Some("conversation.item.create")
@@ -835,13 +957,6 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
_ => None,
})
.await;
eprintln!(
"[realtime test +{}ms] saw audio out data={} sample_rate={} num_channels={}",
start.elapsed().as_millis(),
audio_out.data,
audio_out.sample_rate,
audio_out.num_channels
);
assert_eq!(audio_out.data, "AQID");
let completion = completions
@@ -852,10 +967,6 @@ async fn delegated_turn_user_role_echo_does_not_redelegate_and_still_forwards_au
completion
.await
.expect("delegated turn request did not complete");
eprintln!(
"[realtime test +{}ms] delegated completion resolved",
start.elapsed().as_millis()
);
wait_for_event(&test.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
@@ -298,19 +298,6 @@ async fn request_user_input_rejected_in_default_mode_by_default() -> anyhow::Res
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn request_user_input_rejected_in_realtime_mode() -> anyhow::Result<()> {
assert_request_user_input_rejected("Realtime", |model| CollaborationMode {
mode: ModeKind::Realtime,
settings: Settings {
model,
reasoning_effort: None,
developer_instructions: None,
},
})
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn request_user_input_round_trip_in_default_mode_with_feature() -> anyhow::Result<()> {
request_user_input_round_trip_for_mode(ModeKind::Default).await
-12
View File
@@ -173,7 +173,6 @@ pub enum AltScreenMode {
#[serde(rename_all = "snake_case")]
pub enum ModeKind {
Plan,
Realtime,
#[default]
#[serde(
alias = "code",
@@ -200,7 +199,6 @@ impl ModeKind {
pub const fn display_name(self) -> &'static str {
match self {
Self::Plan => "Plan",
Self::Realtime => "Realtime",
Self::Default => "Default",
Self::PairProgramming => "Pair Programming",
Self::Execute => "Execute",
@@ -348,15 +346,6 @@ mod tests {
}
}
#[test]
fn mode_kind_serializes_and_deserializes_realtime() {
let json = serde_json::to_string(&ModeKind::Realtime).expect("serialize mode");
assert_eq!(json, "\"realtime\"");
let mode: ModeKind = serde_json::from_str(&json).expect("deserialize mode");
assert_eq!(mode, ModeKind::Realtime);
}
#[test]
fn tui_visible_collaboration_modes_match_mode_kind_visibility() {
let expected = [ModeKind::Default, ModeKind::Plan];
@@ -366,7 +355,6 @@ mod tests {
assert!(mode.is_tui_visible());
}
assert!(!ModeKind::Realtime.is_tui_visible());
assert!(!ModeKind::PairProgramming.is_tui_visible());
assert!(!ModeKind::Execute.is_tui_visible());
}
+1 -4
View File
@@ -6990,10 +6990,7 @@ impl ChatWidget {
}
match self.active_mode_kind() {
ModeKind::Plan => Some(CollaborationModeIndicator::Plan),
ModeKind::Default
| ModeKind::Realtime
| ModeKind::PairProgramming
| ModeKind::Execute => None,
ModeKind::Default | ModeKind::PairProgramming | ModeKind::Execute => None,
}
}