diff --git a/codex-rs/core/src/context/realtime_delegation.rs b/codex-rs/core/src/context/realtime_delegation.rs index 5853544fd3..ea896571ca 100644 --- a/codex-rs/core/src/context/realtime_delegation.rs +++ b/codex-rs/core/src/context/realtime_delegation.rs @@ -88,6 +88,17 @@ fn escape_xml_text_bounded(input: &str, retain: Retain) -> String { if escaped.len() <= RealtimeDelegation::FIELD_BYTES { return escaped; } + let range = retained_range(&escaped, retain); + match retain { + Retain::Start => format!("{}{TRUNCATION_MARKER}", &escaped[range]), + Retain::End => format!("{TRUNCATION_MARKER}{}", &escaped[range]), + } +} + +fn retained_range(escaped: &str, retain: Retain) -> std::ops::Range { + if escaped.len() <= RealtimeDelegation::FIELD_BYTES { + return 0..escaped.len(); + } let retained_bytes = RealtimeDelegation::FIELD_BYTES - TRUNCATION_MARKER.len(); match retain { Retain::Start => { @@ -95,14 +106,14 @@ fn escape_xml_text_bounded(input: &str, retain: Retain) -> String { while !escaped.is_char_boundary(end) { end -= 1; } - format!("{}{TRUNCATION_MARKER}", &escaped[..end]) + 0..end } Retain::End => { let mut start = escaped.len() - retained_bytes; while !escaped.is_char_boundary(start) { start += 1; } - format!("{TRUNCATION_MARKER}{}", &escaped[start..]) + start..escaped.len() } } } diff --git a/codex-rs/core/src/context_manager/history.rs b/codex-rs/core/src/context_manager/history.rs index 9e2624bf03..f66cd297a5 100644 --- a/codex-rs/core/src/context_manager/history.rs +++ b/codex-rs/core/src/context_manager/history.rs @@ -1120,10 +1120,7 @@ impl ContextManager { fn get_non_last_reasoning_items_tokens(&self) -> i64 { // Get reasoning items excluding all the ones after the last instruction boundary. - let Some(last_user_index) = self - .items - .iter() - .rposition(|envelope| is_user_turn_boundary(&envelope.item)) + let Some(last_user_index) = self.items.iter().rposition(is_instruction_turn_boundary) else { return 0; }; @@ -1649,6 +1646,11 @@ pub(crate) fn is_user_turn_boundary(item: &ResponseItem) -> bool { || (role == "assistant" && is_inter_agent_instruction_content(content)) } +/// Evaluates an instruction boundary from annotated history. +pub(crate) fn is_instruction_turn_boundary(envelope: &ResponseItemEnvelope) -> bool { + is_user_turn_boundary(&envelope.item) +} + fn is_inter_agent_instruction_content(content: &[ContentItem]) -> bool { InterAgentCommunication::is_message_content(content) } @@ -1656,7 +1658,7 @@ fn is_inter_agent_instruction_content(content: &[ContentItem]) -> bool { fn user_message_positions(items: &[ResponseItemEnvelope]) -> Vec { let mut positions = Vec::new(); for (idx, envelope) in items.iter().enumerate() { - if is_user_turn_boundary(&envelope.item) { + if is_instruction_turn_boundary(envelope) { positions.push(idx); } } diff --git a/codex-rs/core/src/context_manager/mod.rs b/codex-rs/core/src/context_manager/mod.rs index 4621b8295b..ac13ab68f8 100644 --- a/codex-rs/core/src/context_manager/mod.rs +++ b/codex-rs/core/src/context_manager/mod.rs @@ -9,6 +9,7 @@ pub(crate) use history::HistoryReplacement; pub(crate) use history::estimate_image_reference_bytes; pub(crate) use history::estimate_item_token_count; pub(crate) use history::estimate_item_token_count_with_audio; +pub(crate) use history::is_instruction_turn_boundary; pub(crate) use history::is_user_turn_boundary; pub(crate) use history::rejected_audio; pub(crate) use media_availability::replace_unresolvable_media_files; diff --git a/codex-rs/core/src/observation_clock.rs b/codex-rs/core/src/observation_clock.rs index 0f89195256..ac7b0123e8 100644 --- a/codex-rs/core/src/observation_clock.rs +++ b/codex-rs/core/src/observation_clock.rs @@ -11,13 +11,14 @@ impl ObservationClock { pub(crate) fn restore(&mut self, items: &[RolloutItem]) { let floor = items .iter() - .filter_map(|item| match item { - RolloutItem::RealtimeItem(item) => item.observed_at_ms, - RolloutItem::Compacted(compacted) => compacted - .resume_metadata - .as_ref() - .and_then(|metadata| metadata.observation_clock_floor_ms), - _ => None, + .filter_map(|item| { + item.realtime_observed_at_ms().or_else(|| match item { + RolloutItem::Compacted(compacted) => compacted + .resume_metadata + .as_ref() + .and_then(|metadata| metadata.observation_clock_floor_ms), + _ => None, + }) }) .max(); if let Some(floor) = floor { diff --git a/codex-rs/core/src/session/handlers.rs b/codex-rs/core/src/session/handlers.rs index 40b9fad8c3..197c40b279 100644 --- a/codex-rs/core/src/session/handlers.rs +++ b/codex-rs/core/src/session/handlers.rs @@ -50,7 +50,7 @@ use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::request_user_input::RequestUserInputResponse; use codex_thread_store::PersistContext; -use crate::context_manager::is_user_turn_boundary; +use crate::context_manager::is_instruction_turn_boundary; use codex_protocol::dynamic_tools::DynamicToolResponse; use codex_protocol::mcp::RequestId as ProtocolRequestId; use codex_rmcp_client::ElicitationAction; @@ -432,8 +432,9 @@ pub async fn shutdown(sess: &Arc, sub_id: String) -> bool { info!("Shutting down Codex instance"); let history = sess.clone_history().await; let turn_count = history - .raw_items() - .filter(|item| is_user_turn_boundary(item)) + .annotated_items() + .iter() + .filter(|item| is_instruction_turn_boundary(item)) .count(); sess.services.session_telemetry.counter( "codex.conversation.turn.count", diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 9eb4d831c7..fa3208718f 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -278,6 +278,7 @@ mod model_catalog_settings; mod model_ids; pub(crate) mod multi_agents; mod plugin_selection; +mod realtime_handoff; mod realtime_history; mod retained_context; mod review; diff --git a/codex-rs/core/src/session/realtime_handoff.rs b/codex-rs/core/src/session/realtime_handoff.rs new file mode 100644 index 0000000000..04c2aa03e7 --- /dev/null +++ b/codex-rs/core/src/session/realtime_handoff.rs @@ -0,0 +1,68 @@ +//! Routes realtime input through ordinary turn admission. + +use super::session::Session; +use super::turn_input; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; +use codex_protocol::protocol::Event; +use codex_protocol::protocol::EventMsg; +use codex_protocol::turn_input::NotSubmittedReason; +use codex_protocol::turn_input::TurnInputMode; +use codex_protocol::turn_input::TurnInputRequest; +use codex_protocol::turn_input::TurnInputSubmission; +use codex_protocol::turn_input::TurnStartOptions; +use codex_user_input::ResolvedUserInput; +use std::sync::Arc; +use uuid::Uuid; + +impl Session { + pub(crate) async fn route_realtime_text_input( + self: &Arc, + text: String, + ) -> Result<(), &'static str> { + let submission_id = Uuid::now_v7().to_string(); + let submission = turn_input::handle( + self, + TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start( + TurnStartOptions { + turn_trigger: Some("realtime".to_string()), + ..Default::default() + }, + ), + TurnInputMode::StartOrSteer, + submission_id.clone(), + ) + .await; + match submission { + Ok(TurnInputSubmission::Accepted) => { + unreachable!("realtime user input returns a turn outcome") + } + Ok(TurnInputSubmission::Started { .. } | TurnInputSubmission::Steered { .. }) => {} + Ok(TurnInputSubmission::NotSubmitted { + reason: NotSubmittedReason::ServerDraining, + }) => { + return Err("Server is draining; retry the turn after reconnecting"); + } + Ok(TurnInputSubmission::NotSubmitted { reason }) => { + self.send_event_raw(Event { + id: submission_id, + msg: EventMsg::Error(ErrorEvent { + misalignment: None, + message: format!("failed to submit turn input: {reason:?}"), + codex_error_info: Some(CodexErrorInfo::BadRequest), + rejected_audio_file_ids: Vec::new(), + }), + }) + .await; + } + Err(error) => { + self.send_event_raw(Event { + id: submission_id, + msg: EventMsg::Error(error.to_error_event(/*message_prefix*/ None)), + }) + .await; + } + } + Ok(()) + } +} diff --git a/codex-rs/core/src/session/rollout_reconstruction.rs b/codex-rs/core/src/session/rollout_reconstruction.rs index dc5e73aa93..3754124caa 100644 --- a/codex-rs/core/src/session/rollout_reconstruction.rs +++ b/codex-rs/core/src/session/rollout_reconstruction.rs @@ -1,6 +1,6 @@ use super::*; use crate::context::world_state::WorldStateSnapshot; -use crate::context_manager::is_user_turn_boundary; +use crate::context_manager::is_instruction_turn_boundary; use codex_history::ResponseItemEnvelope; use codex_history::TurnAttribution; use codex_protocol::ContextWindowId; @@ -459,7 +459,7 @@ impl ContextManager { let active_segment = active_segment.get_or_insert_with(ActiveReplaySegment::default); active_segment.counts_as_user_turn |= - is_user_turn_boundary(&response_item.item); + is_instruction_turn_boundary(response_item); } RolloutItem::InterAgentCommunication(_) => { let active_segment = diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index 8f9eb5e600..b975ab6866 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -40,10 +40,6 @@ use codex_protocol::error::Result as CodexResult; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AdditionalContextEntry; use codex_protocol::protocol::AdditionalContextKind; -use codex_protocol::protocol::CodexErrorInfo; -use codex_protocol::protocol::ErrorEvent; -use codex_protocol::protocol::Event; -use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::NonSteerableTurnKind; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; @@ -60,7 +56,6 @@ use codex_user_input::ResolvedUserInput; use serde_json::Value; use std::collections::BTreeMap; use std::sync::Arc; -use uuid::Uuid; #[path = "turn_input_agent.rs"] mod agent; @@ -926,56 +921,6 @@ impl Session { && TurnStartKind::Automatic.permits_mode(config.step_settings.collaboration_mode.mode) } - pub(crate) async fn route_realtime_text_input( - self: &Arc, - text: String, - ) -> Result<(), &'static str> { - let submission_id = Uuid::now_v7().to_string(); - let submission = handle( - self, - TurnInputRequest::user_input(vec![ResolvedUserInput::text(text)]).on_start( - TurnStartOptions { - turn_trigger: Some("realtime".to_string()), - ..Default::default() - }, - ), - TurnInputMode::StartOrSteer, - submission_id.clone(), - ) - .await; - match submission { - Ok(TurnInputSubmission::Accepted) => { - unreachable!("realtime user input returns a turn outcome") - } - Ok(TurnInputSubmission::Started { .. } | TurnInputSubmission::Steered { .. }) => {} - Ok(TurnInputSubmission::NotSubmitted { - reason: NotSubmittedReason::ServerDraining, - }) => { - return Err("Server is draining; retry the turn after reconnecting"); - } - Ok(TurnInputSubmission::NotSubmitted { reason }) => { - self.send_event_raw(Event { - id: submission_id, - msg: EventMsg::Error(ErrorEvent { - misalignment: None, - message: format!("failed to submit turn input: {reason:?}"), - codex_error_info: Some(CodexErrorInfo::BadRequest), - rejected_audio_file_ids: Vec::new(), - }), - }) - .await; - } - Err(error) => { - self.send_event_raw(Event { - id: submission_id, - msg: EventMsg::Error(error.to_error_event(/*message_prefix*/ None)), - }) - .await; - } - } - Ok(()) - } - async fn clear_reserved_idle_turn(&self, turn_state: &Arc>) { let mut active_turn_guard = self.active_turn.lock().await; if let Some(active_turn) = active_turn_guard.as_ref() diff --git a/codex-rs/core/src/session/turn_input_tests.rs b/codex-rs/core/src/session/turn_input_tests.rs index 3d37a02ecb..95d2fcf795 100644 --- a/codex-rs/core/src/session/turn_input_tests.rs +++ b/codex-rs/core/src/session/turn_input_tests.rs @@ -19,6 +19,9 @@ use codex_protocol::models::ContentItem; use codex_protocol::models::ResponseItem; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; +use codex_protocol::protocol::ErrorEvent; +use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::ThreadSettingsOverrides; use codex_protocol::protocol::TurnAbortReason; @@ -29,6 +32,7 @@ use pretty_assertions::assert_eq; use test_case::test_case; use tokio::time::sleep; use tokio_util::sync::CancellationToken; +use uuid::Uuid; #[derive(Clone, Copy)] struct NeverEndingTask { diff --git a/codex-rs/core/src/thread_rollout_truncation.rs b/codex-rs/core/src/thread_rollout_truncation.rs index 5dde1d139d..5d3b9fbb13 100644 --- a/codex-rs/core/src/thread_rollout_truncation.rs +++ b/codex-rs/core/src/thread_rollout_truncation.rs @@ -3,7 +3,7 @@ //! In core, "user turns" are detected by scanning `ResponseItem::Message` items and //! interpreting them via `event_mapping::parse_turn_item(...)`. -use crate::context_manager::is_user_turn_boundary; +use crate::context_manager::is_instruction_turn_boundary; use crate::event_mapping; use codex_app_server_protocol::TurnStatus; use codex_app_server_protocol::build_turns_from_rollout_items; @@ -21,12 +21,12 @@ pub(crate) fn initial_history_has_prior_user_turns(conversation_history: &Initia fn rollout_item_is_user_turn_boundary(item: &RolloutItem) -> bool { match item { - RolloutItem::ResponseItem(item) => is_user_turn_boundary(item), + RolloutItem::ResponseItem(item) => is_instruction_turn_boundary(item), RolloutItem::Compacted(checkpoint) => checkpoint .replacement_history .iter() .flatten() - .any(|item| is_user_turn_boundary(item)), + .any(is_instruction_turn_boundary), RolloutItem::InterAgentCommunication(_) => true, _ => false, } diff --git a/codex-rs/history/src/lib.rs b/codex-rs/history/src/lib.rs index 769b2562b8..7758f0b286 100644 --- a/codex-rs/history/src/lib.rs +++ b/codex-rs/history/src/lib.rs @@ -19,6 +19,7 @@ mod initialization; pub use initialization::HistoryInitialization; use std::borrow::Borrow; +use std::borrow::Cow; use std::ops::Deref; use std::ops::DerefMut; use std::path::PathBuf; @@ -331,6 +332,24 @@ pub enum RolloutItem { RealtimeItem(RealtimeItem), } +impl RolloutItem { + /// Returns the canonical realtime observation time without constructing a projection. + pub fn realtime_observed_at_ms(&self) -> Option { + match self { + Self::RealtimeItem(item) => item.observed_at_ms, + _ => None, + } + } + + /// Returns the realtime presentation of this canonical rollout item, if any. + pub fn realtime_item(&self) -> Option> { + match self { + Self::RealtimeItem(item) => Some(Cow::Borrowed(item)), + _ => None, + } + } +} + impl Serialize for RolloutItem { fn serialize(&self, serializer: S) -> Result where diff --git a/codex-rs/rollout/src/recorder.rs b/codex-rs/rollout/src/recorder.rs index b4b8bda7f9..3bc16c9bbd 100644 --- a/codex-rs/rollout/src/recorder.rs +++ b/codex-rs/rollout/src/recorder.rs @@ -1793,9 +1793,9 @@ struct RolloutWriterState { impl RolloutWriterState { fn add_items(&mut self, items: Vec) { - self.realtime_sync_pending |= items.iter().any( - |item| matches!(item, RolloutItem::RealtimeItem(item) if item.observed_at_ms.is_some()), - ); + self.realtime_sync_pending |= items + .iter() + .any(|item| item.realtime_observed_at_ms().is_some()); self.pending_items.extend(items); } diff --git a/codex-rs/thread-store/src/local/rollout_migration.rs b/codex-rs/thread-store/src/local/rollout_migration.rs index 968af560a0..da9243e35b 100644 --- a/codex-rs/thread-store/src/local/rollout_migration.rs +++ b/codex-rs/thread-store/src/local/rollout_migration.rs @@ -1220,11 +1220,10 @@ impl LocalThreadStore { } else { project_rollout_line(&line) }, - realtime_item: match line.item { - RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => { - Some(item) - } - _ => None, + realtime_item: if is_inherited_subagent_history { + None + } else { + line.item.realtime_item().map(std::borrow::Cow::into_owned) }, }, ))); diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback.rs index b2ce0a1c3e..0446a64a0c 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback.rs @@ -14,14 +14,15 @@ use codex_protocol::models::ResponseItem; use codex_protocol::protocol::APPS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::CATALOG_APPS_INSTRUCTIONS_OPEN_TAG; use codex_protocol::protocol::InterAgentCommunication; -use std::borrow::Borrow; +use codex_rollout::ResponseItemEnvelope; /// Match the rollback boundaries used by legacy history reconstruction without /// treating every persisted lifecycle as a user turn. This intentionally stays /// local to the frozen migration adapter: the full core predicate also knows /// about dynamically registered contextual fragments that thread-store cannot /// depend on. -pub(super) fn counts_as_boundary(response: &ResponseItem) -> bool { +pub(super) fn counts_as_boundary(envelope: &ResponseItemEnvelope) -> bool { + let response = &envelope.item; if matches!(response, ResponseItem::AgentMessage { .. }) { return true; } @@ -43,17 +44,14 @@ pub(super) fn is_pre_turn_context_update(response: &ResponseItem) -> bool { /// Apply the same response-history cut used by legacy cold resume to a persisted /// compaction checkpoint. The migration adapter uses a frozen contextual-fragment /// matcher because thread-store cannot depend on core's runtime fragment registry. -pub(super) fn drop_last_n_user_turns(history: &mut Vec, num_turns: u32) -where - T: Borrow, -{ +pub(super) fn drop_last_n_user_turns(history: &mut Vec, num_turns: u32) { if num_turns == 0 { return; } let user_positions = history .iter() .enumerate() - .filter_map(|(index, item)| counts_as_boundary(item.borrow()).then_some(index)) + .filter_map(|(index, item)| counts_as_boundary(item).then_some(index)) .collect::>(); let Some(&first_turn_index) = user_positions.first() else { return; @@ -64,9 +62,7 @@ where } else { user_positions[user_positions.len() - count] }; - while cut_index > first_turn_index - && is_pre_turn_context_update(history[cut_index - 1].borrow()) - { + while cut_index > first_turn_index && is_pre_turn_context_update(&history[cut_index - 1].item) { cut_index -= 1; } history.truncate(cut_index); diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs index 12ae33c540..a017d820f7 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_plan.rs @@ -182,7 +182,7 @@ impl RollbackPlanner { self.record_boundaries[index] = Some(boundary); self.boundaries[boundary].message_id = response.id().cloned(); self.boundaries[boundary].input_source = response.metadata.as_ref().into(); - } else if rollback::counts_as_boundary(&response.item) { + } else if rollback::counts_as_boundary(response) { let boundary = self.start_boundary(index); self.boundaries[boundary].message_id = response.id().cloned(); self.boundaries[boundary].input_source = response.metadata.as_ref().into(); diff --git a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs index 92a0c6c0b2..33d521f13d 100644 --- a/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs +++ b/codex-rs/thread-store/src/local/rollout_migration/rollback_replay.rs @@ -68,7 +68,7 @@ impl ModelReplayPlanner { RolloutItem::TurnContext(context) => ReplayRecord::TurnContext(context.turn_id.clone()), RolloutItem::EventMsg(EventMsg::UserMessage(_)) | RolloutItem::InterAgentCommunication(_) => ReplayRecord::UserBoundary, - RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(&response.item) => { + RolloutItem::ResponseItem(response) if rollback::counts_as_boundary(response) => { ReplayRecord::UserBoundary } RolloutItem::DynamicToolsUpdated { .. } diff --git a/codex-rs/thread-store/src/local/thread_history_materialization.rs b/codex-rs/thread-store/src/local/thread_history_materialization.rs index 8367e57345..86e924c020 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization.rs @@ -12,7 +12,6 @@ use chrono::DateTime; use codex_app_server_protocol::ThreadHistoryChangeSet; use codex_app_server_protocol::project_rollout_line; use codex_protocol::ThreadId; -use codex_rollout::RolloutItem; use tracing::warn; use super::LocalThreadStore; @@ -318,12 +317,16 @@ fn read_projection_batches( } else { project_rollout_line(&line) }; + let realtime_item = if is_inherited_subagent_history { + None + } else { + line.item.realtime_item().map(std::borrow::Cow::into_owned) + }; let fallback_created_at_ms = if changes .changed_items .iter() .any(|change| change.item.started_at_ms.is_none()) - || (!is_inherited_subagent_history - && matches!(&line.item, RolloutItem::RealtimeItem(_))) + || realtime_item.is_some() { match DateTime::parse_from_rfc3339(line.timestamp.as_str()) { Ok(timestamp) => Some(timestamp.timestamp_millis()), @@ -389,10 +392,7 @@ fn read_projection_batches( end_byte_offset: line_end_offset, fallback_created_at_ms, changes, - realtime_item: match line.item { - RolloutItem::RealtimeItem(item) if !is_inherited_subagent_history => Some(item), - _ => None, - }, + realtime_item, }, ))); next_ordinal = next_line_ordinal;