Add internal realtime collaboration mode

This commit is contained in:
Ahmed Ibrahim
2026-02-27 10:16:48 -08:00
parent 3b5996f988
commit faeec29511
12 changed files with 348 additions and 33 deletions
+112
View File
@@ -3035,6 +3035,47 @@ impl Session {
state.session_configuration.collaboration_mode.clone()
}
pub(crate) async fn enter_realtime_collaboration_mode(&self) -> ConstraintResult<()> {
let current = self.collaboration_mode().await;
let collaboration_mode = self
.services
.models_manager
.list_collaboration_modes()
.into_iter()
.find(|mask| mask.mode == Some(ModeKind::Realtime))
.map(|mask| current.apply_mask(&mask))
.unwrap_or_else(|| CollaborationMode {
mode: ModeKind::Realtime,
settings: Settings {
model: current.model().to_string(),
reasoning_effort: current.reasoning_effort(),
developer_instructions: None,
},
});
self.update_settings(SessionSettingsUpdate {
collaboration_mode: Some(collaboration_mode),
..Default::default()
})
.await
}
pub(crate) async fn reset_to_default_collaboration_mode(&self) -> ConstraintResult<()> {
let current = self.collaboration_mode().await;
self.update_settings(SessionSettingsUpdate {
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: current.model().to_string(),
reasoning_effort: current.reasoning_effort(),
developer_instructions: None,
},
}),
..Default::default()
})
.await
}
async fn send_raw_response_items(&self, turn_context: &TurnContext, items: &[ResponseItem]) {
for item in items {
self.send_event(
@@ -4551,6 +4592,9 @@ mod handlers {
pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
sess.abort_all_tasks(TurnAbortReason::Interrupted).await;
let _ = sess.conversation.shutdown().await;
if let Err(err) = sess.reset_to_default_collaboration_mode().await {
warn!("failed to reset collaboration mode during shutdown: {err}");
}
sess.services
.unified_exec_manager
.terminate_all_processes()
@@ -6594,6 +6638,8 @@ mod tests {
use codex_protocol::models::FunctionCallOutputBody;
use codex_protocol::models::FunctionCallOutputPayload;
use crate::protocol::COLLABORATION_MODE_CLOSE_TAG;
use crate::protocol::COLLABORATION_MODE_OPEN_TAG;
use crate::protocol::CompactedItem;
use crate::protocol::CreditsSnapshot;
use crate::protocol::InitialHistory;
@@ -8922,6 +8968,72 @@ mod tests {
assert!(environment_update.contains("<timezone>Europe/Berlin</timezone>"));
}
#[tokio::test]
async fn build_settings_update_items_emits_empty_collaboration_marker_when_instructions_clear()
{
let (session, previous_context) = make_session_and_context().await;
let previous_context = Arc::new(previous_context);
let mut current_context = previous_context
.with_model(
previous_context.model_info.slug.clone(),
&session.services.models_manager,
)
.await;
current_context.collaboration_mode = CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: previous_context.collaboration_mode.model().to_string(),
reasoning_effort: previous_context.collaboration_mode.reasoning_effort(),
developer_instructions: None,
},
};
let mut previous_context_item = previous_context.to_turn_context_item();
previous_context_item.collaboration_mode = Some(CollaborationMode {
mode: ModeKind::Realtime,
settings: Settings {
model: previous_context.collaboration_mode.model().to_string(),
reasoning_effort: previous_context.collaboration_mode.reasoning_effort(),
developer_instructions: Some("realtime instructions".to_string()),
},
});
let update_items = session.build_settings_update_items(
Some(&previous_context_item),
None,
&current_context,
);
let developer_update = update_items
.iter()
.find_map(|item| match item {
ResponseItem::Message { role, content, .. } if role == "developer" => {
let [ContentItem::InputText { text }] = content.as_slice() else {
return None;
};
Some(text)
}
_ => None,
})
.expect("developer update item should be emitted");
assert!(developer_update.contains(&format!(
"{COLLABORATION_MODE_OPEN_TAG}{COLLABORATION_MODE_CLOSE_TAG}"
)));
}
#[tokio::test]
async fn shutdown_resets_realtime_collaboration_mode_to_default() {
let (session, _turn_context, _rx) = make_session_and_context_with_rx().await;
session
.enter_realtime_collaboration_mode()
.await
.expect("enter realtime collaboration mode");
assert_eq!(session.collaboration_mode().await.mode, ModeKind::Realtime);
assert!(handlers::shutdown(&session, "shutdown-test".to_string()).await);
assert_eq!(session.collaboration_mode().await.mode, ModeKind::Default);
}
#[tokio::test]
async fn record_context_updates_and_set_reference_context_item_injects_full_context_when_baseline_missing()
{
+12 -5
View File
@@ -8,6 +8,8 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::DeveloperInstructions;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::protocol::COLLABORATION_MODE_CLOSE_TAG;
use codex_protocol::protocol::COLLABORATION_MODE_OPEN_TAG;
use codex_protocol::protocol::TurnContextItem;
fn build_environment_update_item(
@@ -54,16 +56,21 @@ fn build_collaboration_mode_update_item(
) -> Option<DeveloperInstructions> {
let prev = previous?;
if prev.collaboration_mode.as_ref() != Some(&next.collaboration_mode) {
// If the next mode has empty developer instructions, this returns None and we emit no
// update, so prior collaboration instructions remain in the prompt history.
Some(DeveloperInstructions::from_collaboration_mode(
&next.collaboration_mode,
)?)
Some(
DeveloperInstructions::from_collaboration_mode(&next.collaboration_mode)
.unwrap_or_else(empty_collaboration_mode_update),
)
} else {
None
}
}
fn empty_collaboration_mode_update() -> DeveloperInstructions {
DeveloperInstructions::new(format!(
"{COLLABORATION_MODE_OPEN_TAG}{COLLABORATION_MODE_CLOSE_TAG}"
))
}
fn build_personality_update_item(
previous: Option<&TurnContextItem>,
next: &TurnContext,
@@ -6,6 +6,8 @@ use codex_protocol::openai_models::ReasoningEffort;
const COLLABORATION_MODE_PLAN: &str = include_str!("../../templates/collaboration_mode/plan.md");
const COLLABORATION_MODE_DEFAULT: &str =
include_str!("../../templates/collaboration_mode/default.md");
const COLLABORATION_MODE_REALTIME: &str =
include_str!("../../templates/collaboration_mode/realtime.md");
const KNOWN_MODE_NAMES_PLACEHOLDER: &str = "{{KNOWN_MODE_NAMES}}";
const REQUEST_USER_INPUT_AVAILABILITY_PLACEHOLDER: &str = "{{REQUEST_USER_INPUT_AVAILABILITY}}";
const ASKING_QUESTIONS_GUIDANCE_PLACEHOLDER: &str = "{{ASKING_QUESTIONS_GUIDANCE}}";
@@ -24,7 +26,11 @@ pub struct CollaborationModesConfig {
pub(crate) fn builtin_collaboration_mode_presets(
collaboration_modes_config: CollaborationModesConfig,
) -> Vec<CollaborationModeMask> {
vec![plan_preset(), default_preset(collaboration_modes_config)]
vec![
plan_preset(),
default_preset(collaboration_modes_config),
realtime_preset(),
]
}
fn plan_preset() -> CollaborationModeMask {
@@ -47,6 +53,16 @@ fn default_preset(collaboration_modes_config: CollaborationModesConfig) -> Colla
}
}
fn realtime_preset() -> CollaborationModeMask {
CollaborationModeMask {
name: ModeKind::Realtime.display_name().to_string(),
mode: Some(ModeKind::Realtime),
model: None,
reasoning_effort: None,
developer_instructions: Some(Some(COLLABORATION_MODE_REALTIME.to_string())),
}
}
fn default_mode_instructions(collaboration_modes_config: CollaborationModesConfig) -> String {
let known_mode_names = format_mode_names(&TUI_VISIBLE_COLLABORATION_MODES);
let request_user_input_availability = request_user_input_availability_message(
@@ -118,6 +134,8 @@ mod tests {
plan_preset().reasoning_effort,
Some(Some(ReasoningEffort::Medium))
);
assert_eq!(realtime_preset().name, ModeKind::Realtime.display_name());
assert_eq!(realtime_preset().reasoning_effort, None);
}
#[test]
@@ -156,4 +174,19 @@ mod tests {
.contains("ask the user directly with a concise plain-text question")
);
}
#[test]
fn realtime_preset_only_sets_developer_instructions() {
let realtime = realtime_preset();
assert_eq!(realtime.mode, Some(ModeKind::Realtime));
assert_eq!(realtime.model, None);
assert_eq!(realtime.reasoning_effort, None);
assert!(
realtime
.developer_instructions
.expect("realtime preset should include instructions")
.expect("realtime instructions should be set")
.contains("Realtime mode")
);
}
}
+54 -18
View File
@@ -198,6 +198,11 @@ pub(crate) async fn handle_start(
return Ok(());
}
};
if let Err(err) = sess.enter_realtime_collaboration_mode().await {
let _ = sess.conversation.shutdown().await;
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::BadRequest).await;
return Ok(());
}
info!("realtime conversation started");
@@ -238,13 +243,13 @@ pub(crate) async fn handle_start(
}
if let Some(()) = sess_clone.conversation.running_state().await {
info!("realtime conversation transport closed");
sess_clone
.send_event_raw(ev(EventMsg::RealtimeConversationClosed(
RealtimeConversationClosedEvent {
reason: Some("transport_closed".to_string()),
},
)))
.await;
close_realtime_and_reset_mode(
&sess_clone,
ev,
"transport_closed",
Some(sub_id.clone()),
)
.await;
}
});
@@ -298,20 +303,51 @@ pub(crate) async fn handle_text(
}
pub(crate) async fn handle_close(sess: &Arc<Session>, sub_id: String) {
match sess.conversation.shutdown().await {
Ok(()) => {
sess.send_event_raw(Event {
id: sub_id,
msg: EventMsg::RealtimeConversationClosed(RealtimeConversationClosedEvent {
reason: Some("requested".to_string()),
}),
})
.await;
}
Err(err) => {
let error_sub_id = sub_id.clone();
close_realtime_and_reset_mode(
sess,
|msg| Event {
id: sub_id.clone(),
msg,
},
"requested",
Some(error_sub_id),
)
.await;
}
async fn close_realtime_and_reset_mode<F>(
sess: &Arc<Session>,
event_builder: F,
reason: &str,
error_sub_id: Option<String>,
) where
F: Fn(EventMsg) -> Event,
{
if let Err(err) = sess.conversation.shutdown().await {
if let Some(sub_id) = error_sub_id {
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::Other).await;
} else {
warn!("failed to shutdown realtime conversation: {err}");
}
return;
}
if let Err(err) = sess.reset_to_default_collaboration_mode().await {
if let Some(sub_id) = error_sub_id {
send_conversation_error(sess, sub_id, err.to_string(), CodexErrorInfo::Other).await;
} else {
warn!("failed to reset collaboration mode after realtime close: {err}");
}
return;
}
sess.send_event_raw(event_builder(EventMsg::RealtimeConversationClosed(
RealtimeConversationClosedEvent {
reason: Some(reason.to_string()),
},
)))
.await;
}
fn spawn_realtime_input_task(
@@ -132,6 +132,7 @@ mod tests {
fn request_user_input_mode_availability_defaults_to_plan_only() {
assert!(ModeKind::Plan.allows_request_user_input());
assert!(!ModeKind::Default.allows_request_user_input());
assert!(!ModeKind::Realtime.allows_request_user_input());
assert!(!ModeKind::Execute.allows_request_user_input());
assert!(!ModeKind::PairProgramming.allows_request_user_input());
}
@@ -150,6 +151,10 @@ mod tests {
request_user_input_unavailable_message(ModeKind::Default, true),
None
);
assert_eq!(
request_user_input_unavailable_message(ModeKind::Realtime, false),
Some("request_user_input is unavailable in Realtime mode".to_string())
);
assert_eq!(
request_user_input_unavailable_message(ModeKind::Execute, false),
Some("request_user_input is unavailable in Execute mode".to_string())
@@ -0,0 +1,13 @@
# Collaboration Mode: Realtime
You are now in Realtime mode. Any previous instructions for other collaboration modes are no longer active.
Realtime mode is active only while handling an active realtime conversation session.
- Keep responses concise and low-latency.
- Prefer interruption-safe phrasing that works well for live streaming and audio playback.
- Avoid unnecessary preamble or long digressions while realtime is active.
Your active mode changes only when new developer instructions with a different `<collaboration_mode>...</collaboration_mode>` change it.
When the realtime conversation closes, the session returns to Default mode.
@@ -544,6 +544,89 @@ async fn collaboration_mode_update_emits_new_instruction_message_when_mode_chang
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collaboration_mode_update_emits_empty_marker_when_instructions_clear() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
let _req1 = mount_sse_once(
&server,
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
)
.await;
let req2 = mount_sse_once(
&server,
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
)
.await;
let test = test_codex().build(&server).await?;
let realtime_text = "realtime mode instructions";
test.codex
.submit(Op::OverrideTurnContext {
cwd: None,
approval_policy: None,
sandbox_policy: None,
windows_sandbox_level: None,
model: None,
effort: None,
summary: None,
collaboration_mode: Some(collab_mode_with_mode_and_instructions(
ModeKind::Realtime,
Some(realtime_text),
)),
personality: None,
})
.await?;
test.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "hello 1".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
})
.await?;
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
test.codex
.submit(Op::OverrideTurnContext {
cwd: None,
approval_policy: None,
sandbox_policy: None,
windows_sandbox_level: None,
model: None,
effort: None,
summary: None,
collaboration_mode: Some(collab_mode_with_mode_and_instructions(
ModeKind::Default,
None,
)),
personality: None,
})
.await?;
test.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "hello 2".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
})
.await?;
wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await;
let input = req2.single_request().input();
let dev_texts = developer_texts(&input);
assert_eq!(count_exact(&dev_texts, &collab_xml(realtime_text)), 1);
assert_eq!(count_exact(&dev_texts, &collab_xml("")), 1);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn collaboration_mode_update_noop_does_not_append_when_mode_is_unchanged() -> Result<()> {
skip_if_no_network!(Ok(()));
@@ -702,7 +785,7 @@ async fn resume_replays_collaboration_instructions() -> Result<()> {
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn empty_collaboration_instructions_are_ignored() -> Result<()> {
async fn empty_collaboration_instructions_emit_empty_marker() -> Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
@@ -749,9 +832,9 @@ async fn empty_collaboration_instructions_are_ignored() -> Result<()> {
let input = req.single_request().input();
let dev_texts = developer_texts(&input);
assert_eq!(dev_texts.len(), 1);
assert_eq!(dev_texts.len(), 2);
let collab_text = collab_xml("");
assert_eq!(count_exact(&dev_texts, &collab_text), 0);
assert_eq!(count_exact(&dev_texts, &collab_text), 1);
Ok(())
}
@@ -76,7 +76,6 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
.await
.unwrap_or_else(|err: ErrorEvent| panic!("conversation start failed: {err:?}"));
assert!(started.session_id.is_some());
let session_created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::SessionCreated { session_id },
@@ -157,7 +156,6 @@ async fn conversation_start_audio_text_close_round_trip() -> Result<()> {
closed.reason.as_deref(),
Some("requested" | "transport_closed")
));
server.shutdown().await;
Ok(())
}
@@ -191,7 +189,6 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> {
.await
.unwrap_or_else(|err: ErrorEvent| panic!("conversation start failed: {err:?}"));
assert!(started.session_id.is_some());
let session_created = wait_for_event_match(&test.codex, |msg| match msg {
EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload: RealtimeEvent::SessionCreated { session_id },
@@ -207,7 +204,6 @@ async fn conversation_transport_close_emits_closed_event() -> Result<()> {
})
.await;
assert_eq!(closed.reason.as_deref(), Some("transport_closed"));
server.shutdown().await;
Ok(())
}
@@ -298,6 +298,19 @@ async fn request_user_input_rejected_in_default_mode_by_default() -> anyhow::Res
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn request_user_input_rejected_in_realtime_mode() -> anyhow::Result<()> {
assert_request_user_input_rejected("Realtime", |model| CollaborationMode {
mode: ModeKind::Realtime,
settings: Settings {
model,
reasoning_effort: None,
developer_instructions: None,
},
})
.await
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn request_user_input_round_trip_in_default_mode_with_feature() -> anyhow::Result<()> {
request_user_input_round_trip_for_mode(ModeKind::Default).await
@@ -1,5 +1,6 @@
---
source: core/tests/suite/model_visible_layout.rs
assertion_line: 361
expression: "format_labeled_requests_snapshot(\"First post-resume turn where resumed config model differs from rollout and personality changes.\",\n&[(\"Last Request Before Resume\", &initial_request),\n(\"First Request After Resume\", &resumed_request),])"
---
Scenario: First post-resume turn where resumed config model differs from rollout and personality changes.
@@ -18,8 +19,9 @@ Scenario: First post-resume turn where resumed config model differs from rollout
[02] <ENVIRONMENT_CONTEXT:cwd=<CWD>>
02:message/user:seed resume history
03:message/assistant:recorded before resume
04:message/developer[2]:
04:message/developer[3]:
[01] <model_switch>\nThe user was previously using a different model. Please continue the conversatio...
[02] <PERMISSIONS_INSTRUCTIONS>
[03] <collaboration_mode></collaboration_mode>
05:message/user:<ENVIRONMENT_CONTEXT:cwd=PRETURN_CONTEXT_DIFF_CWD>
06:message/user:resume and change personality
+12
View File
@@ -173,6 +173,7 @@ pub enum AltScreenMode {
#[serde(rename_all = "snake_case")]
pub enum ModeKind {
Plan,
Realtime,
#[default]
#[serde(
alias = "code",
@@ -199,6 +200,7 @@ impl ModeKind {
pub const fn display_name(self) -> &'static str {
match self {
Self::Plan => "Plan",
Self::Realtime => "Realtime",
Self::Default => "Default",
Self::PairProgramming => "Pair Programming",
Self::Execute => "Execute",
@@ -346,6 +348,15 @@ mod tests {
}
}
#[test]
fn mode_kind_serializes_and_deserializes_realtime() {
let json = serde_json::to_string(&ModeKind::Realtime).expect("serialize mode");
assert_eq!(json, "\"realtime\"");
let mode: ModeKind = serde_json::from_str(&json).expect("deserialize mode");
assert_eq!(mode, ModeKind::Realtime);
}
#[test]
fn tui_visible_collaboration_modes_match_mode_kind_visibility() {
let expected = [ModeKind::Default, ModeKind::Plan];
@@ -355,6 +366,7 @@ mod tests {
assert!(mode.is_tui_visible());
}
assert!(!ModeKind::Realtime.is_tui_visible());
assert!(!ModeKind::PairProgramming.is_tui_visible());
assert!(!ModeKind::Execute.is_tui_visible());
}
+4 -1
View File
@@ -6990,7 +6990,10 @@ impl ChatWidget {
}
match self.active_mode_kind() {
ModeKind::Plan => Some(CollaborationModeIndicator::Plan),
ModeKind::Default | ModeKind::PairProgramming | ModeKind::Execute => None,
ModeKind::Default
| ModeKind::Realtime
| ModeKind::PairProgramming
| ModeKind::Execute => None,
}
}