Refactor realtime handoff and history access (#62245)

## What changed

Move realtime text routing into `realtime_handoff` and keep it on the shared start-or-steer path. Extract UTF-8-safe range selection for realtime delegation truncation.

Use `RolloutItem` accessors for realtime timestamps and items in recording, clock restoration, and thread history reads. Pass annotated response items to turn-boundary checks in core and migration rollback so callers can work directly with history envelopes.

GitOrigin-RevId: 3dd37cf91ee5abdca84e87b7d7e304749d9baf04
This commit is contained in:
Zahan Malkani
2026-11-18 09:55:30 +00:00
committed by copyberry
parent ca21fe65d1
commit 6d9f0199ed
18 changed files with 152 additions and 104 deletions
@@ -88,6 +88,17 @@ fn escape_xml_text_bounded(input: &str, retain: Retain) -> String {
if escaped.len() <= RealtimeDelegation::FIELD_BYTES {
return escaped;
}
let range = retained_range(&escaped, retain);
match retain {
Retain::Start => format!("{}{TRUNCATION_MARKER}", &escaped[range]),
Retain::End => format!("{TRUNCATION_MARKER}{}", &escaped[range]),
}
}
fn retained_range(escaped: &str, retain: Retain) -> std::ops::Range<usize> {
if escaped.len() <= RealtimeDelegation::FIELD_BYTES {
return 0..escaped.len();
}
let retained_bytes = RealtimeDelegation::FIELD_BYTES - TRUNCATION_MARKER.len();
match retain {
Retain::Start => {
@@ -95,14 +106,14 @@ fn escape_xml_text_bounded(input: &str, retain: Retain) -> String {
while !escaped.is_char_boundary(end) {
end -= 1;
}
format!("{}{TRUNCATION_MARKER}", &escaped[..end])
0..end
}
Retain::End => {
let mut start = escaped.len() - retained_bytes;
while !escaped.is_char_boundary(start) {
start += 1;
}
format!("{TRUNCATION_MARKER}{}", &escaped[start..])
start..escaped.len()
}
}
}
+7 -5
View File
@@ -1120,10 +1120,7 @@ impl ContextManager {
fn get_non_last_reasoning_items_tokens(&self) -> i64 {
// Get reasoning items excluding all the ones after the last instruction boundary.
let Some(last_user_index) = self
.items
.iter()
.rposition(|envelope| is_user_turn_boundary(&envelope.item))
let Some(last_user_index) = self.items.iter().rposition(is_instruction_turn_boundary)
else {
return 0;
};
@@ -1649,6 +1646,11 @@ pub(crate) fn is_user_turn_boundary(item: &ResponseItem) -> bool {
|| (role == "assistant" && is_inter_agent_instruction_content(content))
}
/// Evaluates an instruction boundary from annotated history.
pub(crate) fn is_instruction_turn_boundary(envelope: &ResponseItemEnvelope) -> bool {
is_user_turn_boundary(&envelope.item)
}
fn is_inter_agent_instruction_content(content: &[ContentItem]) -> bool {
InterAgentCommunication::is_message_content(content)
}
@@ -1656,7 +1658,7 @@ fn is_inter_agent_instruction_content(content: &[ContentItem]) -> bool {
fn user_message_positions(items: &[ResponseItemEnvelope]) -> Vec<usize> {
let mut positions = Vec::new();
for (idx, envelope) in items.iter().enumerate() {
if is_user_turn_boundary(&envelope.item) {
if is_instruction_turn_boundary(envelope) {
positions.push(idx);
}
}
+1
View File
@@ -9,6 +9,7 @@ pub(crate) use history::HistoryReplacement;
pub(crate) use history::estimate_image_reference_bytes;
pub(crate) use history::estimate_item_token_count;
pub(crate) use history::estimate_item_token_count_with_audio;
pub(crate) use history::is_instruction_turn_boundary;
pub(crate) use history::is_user_turn_boundary;
pub(crate) use history::rejected_audio;
pub(crate) use media_availability::replace_unresolvable_media_files;
+8 -7
View File
@@ -11,13 +11,14 @@ impl ObservationClock {
pub(crate) fn restore(&mut self, items: &[RolloutItem]) {
let floor = items
.iter()
.filter_map(|item| match item {
RolloutItem::RealtimeItem(item) => item.observed_at_ms,
RolloutItem::Compacted(compacted) => compacted
.resume_metadata
.as_ref()
.and_then(|metadata| metadata.observation_clock_floor_ms),
_ => None,
.filter_map(|item| {
item.realtime_observed_at_ms().or_else(|| match item {
RolloutItem::Compacted(compacted) => compacted
.resume_metadata
.as_ref()
.and_then(|metadata| metadata.observation_clock_floor_ms),
_ => None,
})
})
.max();
if let Some(floor) = floor {
+4 -3
View File
@@ -50,7 +50,7 @@ use codex_protocol::request_permissions::RequestPermissionsResponse;
use codex_protocol::request_user_input::RequestUserInputResponse;
use codex_thread_store::PersistContext;
use crate::context_manager::is_user_turn_boundary;
use crate::context_manager::is_instruction_turn_boundary;
use codex_protocol::dynamic_tools::DynamicToolResponse;
use codex_protocol::mcp::RequestId as ProtocolRequestId;
use codex_rmcp_client::ElicitationAction;
@@ -432,8 +432,9 @@ pub async fn shutdown(sess: &Arc<Session>, sub_id: String) -> bool {
info!("Shutting down Codex instance");
let history = sess.clone_history().await;
let turn_count = history
.raw_items()
.filter(|item| is_user_turn_boundary(item))
.annotated_items()
.iter()
.filter(|item| is_instruction_turn_boundary(item))
.count();
sess.services.session_telemetry.counter(
"codex.conversation.turn.count",
+1
View File
@@ -278,6 +278,7 @@ mod model_catalog_settings;
mod model_ids;
pub(crate) mod multi_agents;
mod plugin_selection;
mod realtime_handoff;
mod realtime_history;
mod retained_context;
mod review;
@@ -0,0 +1,68 @@
//! Routes realtime input through ordinary turn admission.
use super::session::Session;
use super::turn_input;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::turn_input::NotSubmittedReason;
use codex_protocol::turn_input::TurnInputMode;
use codex_protocol::turn_input::TurnInputRequest;
use codex_protocol::turn_input::TurnInputSubmission;
use codex_protocol::turn_input::TurnStartOptions;
use codex_user_input::ResolvedUserInput;
use std::sync::Arc;
use uuid::Uuid;
impl Session {
pub(crate) async fn route_realtime_text_input(
self: &Arc<Self>,
text: String,
) -> Result<(), &'static str> {
let submission_id = Uuid::now_v7().to_string();
let submission = turn_input::handle(
self,
TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start(
TurnStartOptions {
turn_trigger: Some("realtime".to_string()),
..Default::default()
},
),
TurnInputMode::StartOrSteer,
submission_id.clone(),
)
.await;
match submission {
Ok(TurnInputSubmission::Accepted) => {
unreachable!("realtime user input returns a turn outcome")
}
Ok(TurnInputSubmission::Started { .. } | TurnInputSubmission::Steered { .. }) => {}
Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::ServerDraining,
}) => {
return Err("Server is draining; retry the turn after reconnecting");
}
Ok(TurnInputSubmission::NotSubmitted { reason }) => {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(ErrorEvent {
misalignment: None,
message: format!("failed to submit turn input: {reason:?}"),
codex_error_info: Some(CodexErrorInfo::BadRequest),
rejected_audio_file_ids: Vec::new(),
}),
})
.await;
}
Err(error) => {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(error.to_error_event(/*message_prefix*/ None)),
})
.await;
}
}
Ok(())
}
}
@@ -1,6 +1,6 @@
use super::*;
use crate::context::world_state::WorldStateSnapshot;
use crate::context_manager::is_user_turn_boundary;
use crate::context_manager::is_instruction_turn_boundary;
use codex_history::ResponseItemEnvelope;
use codex_history::TurnAttribution;
use codex_protocol::ContextWindowId;
@@ -459,7 +459,7 @@ impl ContextManager {
let active_segment =
active_segment.get_or_insert_with(ActiveReplaySegment::default);
active_segment.counts_as_user_turn |=
is_user_turn_boundary(&response_item.item);
is_instruction_turn_boundary(response_item);
}
RolloutItem::InterAgentCommunication(_) => {
let active_segment =
-55
View File
@@ -40,10 +40,6 @@ use codex_protocol::error::Result as CodexResult;
use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::AdditionalContextEntry;
use codex_protocol::protocol::AdditionalContextKind;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::NonSteerableTurnKind;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
@@ -60,7 +56,6 @@ use codex_user_input::ResolvedUserInput;
use serde_json::Value;
use std::collections::BTreeMap;
use std::sync::Arc;
use uuid::Uuid;
#[path = "turn_input_agent.rs"]
mod agent;
@@ -926,56 +921,6 @@ impl Session {
&& TurnStartKind::Automatic.permits_mode(config.step_settings.collaboration_mode.mode)
}
pub(crate) async fn route_realtime_text_input(
self: &Arc<Self>,
text: String,
) -> Result<(), &'static str> {
let submission_id = Uuid::now_v7().to_string();
let submission = handle(
self,
TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start(
TurnStartOptions {
turn_trigger: Some("realtime".to_string()),
..Default::default()
},
),
TurnInputMode::StartOrSteer,
submission_id.clone(),
)
.await;
match submission {
Ok(TurnInputSubmission::Accepted) => {
unreachable!("realtime user input returns a turn outcome")
}
Ok(TurnInputSubmission::Started { .. } | TurnInputSubmission::Steered { .. }) => {}
Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::ServerDraining,
}) => {
return Err("Server is draining; retry the turn after reconnecting");
}
Ok(TurnInputSubmission::NotSubmitted { reason }) => {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(ErrorEvent {
misalignment: None,
message: format!("failed to submit turn input: {reason:?}"),
codex_error_info: Some(CodexErrorInfo::BadRequest),
rejected_audio_file_ids: Vec::new(),
}),
})
.await;
}
Err(error) => {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(error.to_error_event(/*message_prefix*/ None)),
})
.await;
}
}
Ok(())
}
async fn clear_reserved_idle_turn(&self, turn_state: &Arc<tokio::sync::Mutex<TurnState>>) {
let mut active_turn_guard = self.active_turn.lock().await;
if let Some(active_turn) = active_turn_guard.as_ref()
@@ -19,6 +19,9 @@ use codex_protocol::models::ContentItem;
use codex_protocol::models::ResponseItem;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::InterAgentCommunication;
use codex_protocol::protocol::ThreadSettingsOverrides;
use codex_protocol::protocol::TurnAbortReason;
@@ -29,6 +32,7 @@ use pretty_assertions::assert_eq;
use test_case::test_case;
use tokio::time::sleep;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
#[derive(Clone, Copy)]
struct NeverEndingTask {
@@ -3,7 +3,7 @@
//! In core, "user turns" are detected by scanning `ResponseItem::Message` items and
//! interpreting them via `event_mapping::parse_turn_item(...)`.
use crate::context_manager::is_user_turn_boundary;
use crate::context_manager::is_instruction_turn_boundary;
use crate::event_mapping;
use codex_app_server_protocol::TurnStatus;
use codex_app_server_protocol::build_turns_from_rollout_items;
@@ -21,12 +21,12 @@ pub(crate) fn initial_history_has_prior_user_turns(conversation_history: &Initia
fn rollout_item_is_user_turn_boundary(item: &RolloutItem) -> bool {
match item {
RolloutItem::ResponseItem(item) => is_user_turn_boundary(item),
RolloutItem::ResponseItem(item) => is_instruction_turn_boundary(item),
RolloutItem::Compacted(checkpoint) => checkpoint
.replacement_history
.iter()
.flatten()
.any(|item| is_user_turn_boundary(item)),
.any(is_instruction_turn_boundary),
RolloutItem::InterAgentCommunication(_) => true,
_ => false,
}
+19
View File
@@ -19,6 +19,7 @@ mod initialization;
pub use initialization::HistoryInitialization;
use std::borrow::Borrow;
use std::borrow::Cow;
use std::ops::Deref;
use std::ops::DerefMut;
use std::path::PathBuf;
@@ -331,6 +332,24 @@ pub enum RolloutItem {
RealtimeItem(RealtimeItem),
}
impl RolloutItem {
/// Returns the canonical realtime observation time without constructing a projection.
pub fn realtime_observed_at_ms(&self) -> Option<i64> {
match self {
Self::RealtimeItem(item) => item.observed_at_ms,
_ => None,
}
}
/// Returns the realtime presentation of this canonical rollout item, if any.
pub fn realtime_item(&self) -> Option<Cow<'_, RealtimeItem>> {
match self {
Self::RealtimeItem(item) => Some(Cow::Borrowed(item)),
_ => None,
}
}
}
impl Serialize for RolloutItem {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
+3 -3
View File
@@ -1793,9 +1793,9 @@ struct RolloutWriterState {
impl RolloutWriterState {
fn add_items(&mut self, items: Vec<RolloutItem>) {
self.realtime_sync_pending |= items.iter().any(
|item| matches!(item, RolloutItem::RealtimeItem(item) if item.observed_at_ms.is_some()),
);
self.realtime_sync_pending |= items
.iter()
.any(|item| item.realtime_observed_at_ms().is_some());
self.pending_items.extend(items);
}
@@ -1220,11 +1220,10 @@ impl LocalThreadStore {
} else {
project_rollout_line(&line)
},
realtime_item: match line.item {
RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => {
Some(item)
}
_ => None,
realtime_item: if is_inherited_subagent_history {
None
} else {
line.item.realtime_item().map(std::borrow::Cow::into_owned)
},
},
)));
@@ -14,14 +14,15 @@ use codex_protocol::models::ResponseItem;
use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG;
use codex_protocol::protocol::CATALOG_APPS_INSTRUCTIONS_OPEN_TAG;
use codex_protocol::protocol::InterAgentCommunication;
use std::borrow::Borrow;
use codex_rollout::ResponseItemEnvelope;
/// Match the rollback boundaries used by legacy history reconstruction without
/// treating every persisted lifecycle as a user turn. This intentionally stays
/// local to the frozen migration adapter: the full core predicate also knows
/// about dynamically registered contextual fragments that thread-store cannot
/// depend on.
pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool {
pub(super) fn counts_as_boundary(envelope: &ResponseItemEnvelope) -> bool {
let response = &envelope.item;
if matches!(response, ResponseItem::AgentMessage { .. }) {
return true;
}
@@ -43,17 +44,14 @@ pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool {
/// Apply the same response-history cut used by legacy cold resume to a persisted
/// compaction checkpoint. The migration adapter uses a frozen contextual-fragment
/// matcher because thread-store cannot depend on core's runtime fragment registry.
pub(super) fn drop_last_n_user_turns<T>(history: &mut Vec<T>, num_turns: u32)
where
T: Borrow<ResponseItem>,
{
pub(super) fn drop_last_n_user_turns(history: &mut Vec<ResponseItemEnvelope>, num_turns: u32) {
if num_turns == 0 {
return;
}
let user_positions = history
.iter()
.enumerate()
.filter_map(|(index, item)| counts_as_boundary(item.borrow()).then_some(index))
.filter_map(|(index, item)| counts_as_boundary(item).then_some(index))
.collect::<Vec<_>>();
let Some(&first_turn_index) = user_positions.first() else {
return;
@@ -64,9 +62,7 @@ where
} else {
user_positions[user_positions.len() - count]
};
while cut_index > first_turn_index
&& is_pre_turn_context_update(history[cut_index - 1].borrow())
{
while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1].item) {
cut_index -= 1;
}
history.truncate(cut_index);
@@ -182,7 +182,7 @@ impl RollbackPlanner {
self.record_boundaries[index] = Some(boundary);
self.boundaries[boundary].message_id = response.id().cloned();
self.boundaries[boundary].input_source = response.metadata.as_ref().into();
} else if rollback::counts_as_boundary(&response.item) {
} else if rollback::counts_as_boundary(response) {
let boundary = self.start_boundary(index);
self.boundaries[boundary].message_id = response.id().cloned();
self.boundaries[boundary].input_source = response.metadata.as_ref().into();
@@ -68,7 +68,7 @@ impl ModelReplayPlanner {
RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()),
RolloutItem::EventMsg(EventMsg::UserMessage(_))
| RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary,
RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(&response.item) => {
RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => {
ReplayRecord::UserBoundary
}
RolloutItem::DynamicToolsUpdated { .. }
@@ -12,7 +12,6 @@ use chrono::DateTime;
use codex_app_server_protocol::ThreadHistoryChangeSet;
use codex_app_server_protocol::project_rollout_line;
use codex_protocol::ThreadId;
use codex_rollout::RolloutItem;
use tracing::warn;
use super::LocalThreadStore;
@@ -318,12 +317,16 @@ fn read_projection_batches(
} else {
project_rollout_line(&line)
};
let realtime_item = if is_inherited_subagent_history {
None
} else {
line.item.realtime_item().map(std::borrow::Cow::into_owned)
};
let fallback_created_at_ms = if changes
.changed_items
.iter()
.any(|change| change.item.started_at_ms.is_none())
|| (!is_inherited_subagent_history
&& matches!(&line.item, RolloutItem::RealtimeItem(_)))
|| realtime_item.is_some()
{
match DateTime::parse_from_rfc3339(line.timestamp.as_str()) {
Ok(timestamp) => Some(timestamp.timestamp_millis()),
@@ -389,10 +392,7 @@ fn read_projection_batches(
end_byte_offset: line_end_offset,
fallback_created_at_ms,
changes,
realtime_item: match line.item {
RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => Some(item),
_ => None,
},
realtime_item,
},
)));
next_ordinal = next_line_ordinal;