Persist realtime handoffs before accepting turn input (#62288)

## Why

Realtime history needs to record which transcript frames reached the
coding agent and which turn received them. If a handoff fails or the
conversation reconnects, Codex needs to distinguish speech that was
already used from speech that is still available for the next input.

## What changed

- Route handoffs through normal turn admission to start or steer a turn.
  When realtime frame history is enabled, persist the transcript frames
  and the receiving turn's input before publishing acceptance or exposing
  the input to the model. Attribute only transcript frames represented
  in the rendered input.
- Reserve transcript frames across overlapping handoffs and release them
  on definite rejection. Retry an uncertain write with the same input
  and require history recovery if the write still cannot be confirmed.
- On V3 reconnect, restore persisted speech that has not been consumed
  and put it before newly captured speech.

## Testing

Add regression tests for attribution after transcript truncation,
overlapping handoffs, V3 reconnect ordering, and preservation of typed
input when a handoff in the same turn fails before model sampling.

GitOrigin-RevId: 432dd51971f7ced20344ebc7f0c7b007bb849f54
This commit is contained in:
Zahan Malkani
2026-11-18 14:26:53 +00:00
committed by copyberry
parent df975851bf
commit ad5dd4a110
21 changed files with 1608 additions and 92 deletions
@@ -2,6 +2,7 @@ use super::ContextualUserFragment;
use codex_protocol::models::ContentItem;
use codex_protocol::models::ContentItemKind;
use codex_protocol::models::input_text;
use codex_protocol::realtime::RealtimeTranscriptRole;
const TRUNCATION_MARKER: &str = "…";
@@ -22,6 +23,66 @@ pub(crate) struct RealtimeDelegation<'a> {
impl<'a> RealtimeDelegation<'a> {
pub(crate) const FIELD_BYTES: usize = 4 * 1024;
/// Render and attribute the same fields, so omitted transcript rows
/// cannot accidentally appear in the accepted source list.
pub(crate) fn from_events(
input: &str,
entries: &[(Option<&str>, RealtimeTranscriptRole, &str)],
source: RealtimeDelegationSource,
preserve_transcript: bool,
) -> Option<(String, Vec<String>)> {
if input.is_empty() && entries.is_empty() {
return None;
}
let mut transcript = String::new();
let mut ranges = Vec::new();
let mut escaped_len = 0;
for (id, role, text) in entries {
if !transcript.is_empty() {
transcript.push('\n');
escaped_len += 1;
}
let role = match role {
RealtimeTranscriptRole::User => "user",
RealtimeTranscriptRole::Assistant => "assistant",
};
let line = format!("{role}: {text}");
let start = escaped_len;
escaped_len += escape_xml_text(&line).len();
transcript.push_str(&line);
ranges.push((*id, start..escaped_len));
}
let escaped_transcript = escape_xml_text(&transcript);
let delta_start = if preserve_transcript {
0
} else {
retained_range(&escaped_transcript, Retain::End).start
};
let input_end = retained_range(&escaped_transcript, Retain::Start).end;
let input_source = entries
.iter()
.rfind(|(_, role, text)| {
*role == RealtimeTranscriptRole::User && text.trim() == input.trim()
})
.and_then(|(id, _, _)| *id);
let sources = ranges
.into_iter()
.filter(|(id, range)| {
range.end > delta_start
|| (input.is_empty() && range.start < input_end)
|| (id.is_some() && *id == input_source)
})
.filter_map(|(id, _)| id.map(str::to_string))
.collect();
let input = if input.is_empty() { &transcript } else { input };
Some((
RealtimeDelegation::new(input, Some(&transcript), source, preserve_transcript)
.render()
.into_text()?,
sources,
))
}
pub(crate) fn new(
input: &'a str,
transcript_delta: Option<&'a str>,
@@ -124,3 +185,7 @@ fn escape_xml_text(input: &str) -> String {
.replace('<', "&lt;")
.replace('>', "&gt;")
}
#[cfg(test)]
#[path = "realtime_delegation_tests.rs"]
mod tests;
@@ -0,0 +1,56 @@
use super::*;
use pretty_assertions::assert_eq;
#[test]
fn bounded_context_attributes_only_retained_events_in_source_order() {
let middle = "<&é>".repeat(2048);
let tail = "new context".repeat(600);
let entries = [
(Some("query"), RealtimeTranscriptRole::User, "do the thing"),
(
Some("dropped"),
RealtimeTranscriptRole::Assistant,
middle.as_str(),
),
(
Some("tail"),
RealtimeTranscriptRole::Assistant,
tail.as_str(),
),
];
let (text, sources) = RealtimeDelegation::from_events(
"do the thing",
&entries,
RealtimeDelegationSource::Handoff,
/*preserve_transcript*/ false,
)
.unwrap();
assert_eq!(sources, vec!["query", "tail"]);
assert!(!text.contains("&lt;"));
assert!(text.contains("<input>do the thing</input>"));
assert!(text.len() < 9 * 1024);
}
#[test]
fn transcript_fallback_attributes_both_the_input_prefix_and_context_suffix() {
let long = "x".repeat(9000);
let entries = [
(Some("first"), RealtimeTranscriptRole::User, "question"),
(
Some("middle"),
RealtimeTranscriptRole::Assistant,
long.as_str(),
),
(Some("last"), RealtimeTranscriptRole::User, "followup"),
];
let (text, sources) = RealtimeDelegation::from_events(
"",
&entries,
RealtimeDelegationSource::Handoff,
/*preserve_transcript*/ false,
)
.unwrap();
assert_eq!(sources, vec!["first", "middle", "last"]);
assert!(text.contains("<input>user: question"));
assert!(text.contains("user: followup</transcript_delta>"));
}
@@ -642,6 +642,51 @@ impl ContextManager {
}
}
/// Reserves the item's sampling sequence before a durable acceptance attempt.
pub(crate) fn prepare_item_sampling(&mut self, envelope: &mut ResponseItemEnvelope) {
self.sampling.record(envelope);
}
/// Capture a realtime input's retained provenance without copying or appending model history.
pub(crate) fn stage_realtime_input(
&mut self,
envelope: &mut ResponseItemEnvelope,
) -> Arc<RetainedContext> {
let original_context = Arc::clone(&self.retained_context);
let original_revision = self.user_message_revision;
let captured = self.record_retained_message(
&envelope.item,
envelope.metadata.as_ref(),
user_authorization::RetainedMessageSource::Original,
);
if let Some(source) = envelope
.metadata
.as_ref()
.and_then(|metadata| metadata.retained_source.as_ref())
{
if captured.as_ref().is_some_and(|captured| {
captured.id == source.id && captured.complete == source.complete
}) {
Arc::make_mut(&mut self.retained_context).restore_source_revision(source);
}
} else if let Some(source) = captured {
envelope.metadata.get_or_insert_default().retained_source = Some(source);
}
self.user_message_revision = original_revision;
std::mem::replace(&mut self.retained_context, original_context)
}
/// Install staged provenance and append the accepted input only after its seal succeeds.
pub(crate) fn record_sealed_realtime_input(
&mut self,
envelope: &ResponseItemEnvelope,
retained_context: Arc<RetainedContext>,
policy: TruncationPolicy,
) {
self.retained_context = retained_context;
self.replay_annotated_item(envelope, policy);
}
/// Replays persisted originals without assigning new identities to known versions.
pub(crate) fn replay_annotated_item(
&mut self,
@@ -1261,6 +1261,46 @@ fn record_annotated_items_preserves_metadata_while_processing_item(
);
}
#[test]
fn realtime_input_stages_provenance_without_copying_model_history() {
let mut history = create_history_with_items(vec![user_msg("earlier input")]);
let original_items = Arc::clone(&history.items);
let original_retained = history.retained_context().clone();
let mut envelope = ResponseItemEnvelope::new(ResponseItem::Message {
status: None,
encrypted_content: None,
id: Some(ResponseItemId::with_suffix("msg", "realtime")),
role: "user".to_owned(),
content: vec![ContentItem::InputText {
text: "use the smaller logo".to_owned(),
}],
phase: None,
internal_chat_message_metadata_passthrough: Some(InternalChatMessageMetadataPassthrough {
turn_id: Some("realtime-turn".to_owned()),
content_item_kinds: Some(vec![ContentItemKind("user.text".to_owned())]),
..Default::default()
}),
});
history.prepare_item_sampling(&mut envelope);
let staged = history.stage_realtime_input(&mut envelope);
assert!(Arc::ptr_eq(&history.items, &original_items));
assert_eq!(history.retained_context(), &original_retained);
assert!(
envelope
.metadata
.as_ref()
.is_some_and(|metadata| metadata.retained_source.is_some())
);
// A failed seal can retry the exact cached envelope, including its source revision.
drop(staged);
let cached = envelope.clone();
let staged = history.stage_realtime_input(&mut envelope);
assert_eq!(envelope, cached);
history.record_sealed_realtime_input(&envelope, staged, TruncationPolicy::Tokens(10_000));
assert_eq!(history.annotated_items().last(), Some(&cached));
}
#[tokio::test]
async fn for_prompt_annotated_preserves_metadata_while_normalizing_item() {
let envelope = ResponseItemEnvelope {
@@ -344,7 +344,8 @@ async fn compacted_review_restores_originals_once_and_persists_the_request_prefi
},
codex_thread_store::PersistContext::Standard,
)
.await;
.await
.expect("record review input");
let collected = super::super::prompt::collect_guardian_prompt_items_with_parent_turn(
&session,
super::super::prompt::GuardianTranscriptHistory::Retained(
+3 -2
View File
@@ -759,7 +759,7 @@ pub(crate) async fn record_pending_input(
pending_input: TurnInput,
additional_contexts: Vec<codex_hooks::HookContext>,
persist_context: PersistContext,
) {
) -> std::io::Result<()> {
match pending_input {
TurnInput::UserInput {
content,
@@ -774,7 +774,7 @@ pub(crate) async fn record_pending_input(
metadata,
persist_context,
)
.await;
.await?;
}
TurnInput::ResponseItem(item) => {
sess.record_annotated_conversation_items(turn_context, model_info, vec![item])
@@ -809,6 +809,7 @@ pub(crate) async fn record_pending_input(
}
}
record_additional_contexts(sess, turn_context, additional_contexts).await;
Ok(())
}
/// Processes finished async hook results at a safe turn boundary.
+33 -1
View File
@@ -125,6 +125,12 @@ const REALTIME_V2_STEER_ACKNOWLEDGEMENT: &str =
const REALTIME_ACTIVE_RESPONSE_ERROR_PREFIX: &str =
"Conversation already has an active response in progress:";
const REALTIME_SESSION_ENDED_HANDOFF_INSTRUCTION: &str = "The user just ended their realtime session. Here is the remaining handoff/transcript tail. You probably do not have to do anything; acknowledge the handoff unless the transcript itself asks for something.";
const HISTORY_RECOVERY_MESSAGE: &str =
"realtime history recovery required; reload the thread before continuing";
fn history_recovery_error() -> CodexErr {
CodexErr::Fatal(HISTORY_RECOVERY_MESSAGE.into())
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum RealtimeConversationEnd {
@@ -183,6 +189,7 @@ impl RealtimeHandoffAdmission {
pub(crate) struct RealtimeConversationManager {
state: Mutex<RealtimeConversationManagerState>,
recovery_required: AtomicBool,
}
struct RealtimeConversationManagerState {
@@ -608,6 +615,7 @@ impl RealtimeConversationManager {
mode_instructions: None,
context_active: false,
}),
recovery_required: AtomicBool::new(false),
}
}
@@ -632,6 +640,30 @@ impl RealtimeConversationManager {
.and_then(|state| state.realtime_active.load(Ordering::Relaxed).then_some(()))
}
pub(crate) fn history_recovery_required(&self) -> bool {
self.recovery_required.load(Ordering::Acquire)
}
pub(crate) fn check_history_recovery(&self) -> CodexResult<()> {
if self.history_recovery_required() {
Err(history_recovery_error())
} else {
Ok(())
}
}
pub(crate) fn mark_history_recovery_required(&self) {
self.recovery_required.store(true, Ordering::Release);
}
pub(crate) async fn stop_for_history_recovery(&self) {
if self.history_recovery_required()
&& let Some(conversation) = &self.state.lock().await.conversation
{
conversation.stop_token.cancel();
}
}
pub(crate) async fn is_running_v2(&self) -> bool {
let state = self.state.lock().await;
matches!(
@@ -2111,7 +2143,7 @@ fn realtime_text_from_handoff_request(handoff: &RealtimeHandoffRequested) -> Opt
.or_else(|| realtime_transcript_delta_from_handoff(handoff))
}
fn realtime_delegation_from_handoff(
pub(crate) fn realtime_delegation_from_handoff(
handoff: &RealtimeHandoffRequested,
preserve_transcript: bool,
) -> Option<String> {
+21 -2
View File
@@ -2,6 +2,8 @@
//! transcript boundaries and deduplication state with the history reducer.
//! The reducer selects when its effects are persisted relative to their source event.
#[expect(dead_code, reason = "Activated with hangup in the next PR")]
pub(crate) mod handoff;
mod presentation;
use crate::observation_clock::ObservationClock;
@@ -21,6 +23,8 @@ use codex_protocol::user_input::UserInput;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::sync::Arc;
use tokio::sync::Notify;
use uuid::Uuid;
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -97,9 +101,17 @@ enum Continuation {
/// Retains only live session state; durable history is served by the rollout index.
#[derive(Default)]
pub(crate) struct RealtimeHistoryState {
pending_transcript: VecDeque<RealtimeItem>,
recovered_frame_ids: HashSet<String>,
pending_handoff_seals: Vec<RealtimeItem>,
v3_utterances: VecDeque<handoff::V3Utterance>,
handoff_inputs: HashMap<String, handoff::PreparedRealtimeInput>,
handoff_settled: Arc<Notify>,
committed_handoff_ids: HashSet<String>,
pub(crate) observation_clock: ObservationClock,
/// Resolved once at the first call. Unresolved and unenrolled threads write legacy history.
pub(crate) frame_enrollment: Option<bool>,
preserve_transcript: bool,
active_session_id: Option<String>,
active_segments: ActiveTranscriptSegments,
streaming_agent_message: Option<StreamingAgentMessage>,
@@ -455,12 +467,19 @@ impl RealtimeHistoryState {
continuation: Continuation,
) {
if !segment.text.is_empty() {
items.push(RealtimeItem {
let item = RealtimeItem {
id: segment.id,
realtime_session_id: segment.session_id.clone(),
observed_at_ms: segment.observed_at_ms,
content: speech(self.frames_enabled(), segment.role, segment.text),
});
};
if self.frames_enabled() {
if let Some(utterance) = self.v3_utterance(segment.role, /*start*/ true) {
utterance.frames.push(item.id.clone());
}
self.remember_transcript(item.clone());
}
items.push(item);
}
if matches!(continuation, Continuation::Continue) {
self.active_segments
@@ -0,0 +1,476 @@
//! Builds handoff context from the same immutable transcript items served in history.
//! Pending text is only consumed after turn admission and the acceptance barrier succeed.
use super::Continuation;
use super::RealtimeHistoryState;
use crate::context::RealtimeDelegation;
use crate::context::RealtimeDelegationSource;
use codex_history::ResponseItemEnvelope;
use codex_protocol::protocol::RealtimeHandoffRequested;
use codex_protocol::protocol::RealtimeTranscriptEntry;
use codex_protocol::realtime::RealtimeItem;
use codex_protocol::realtime::RealtimeItemContent;
use codex_protocol::realtime::RealtimeTranscriptRole;
use codex_protocol::realtime::SessionInputReason;
use uuid::Uuid;
// Older protocols retain a bounded transcript tail; V3 drains whole completed utterances.
const PENDING_TRANSCRIPT_BYTES: usize = 64 * 1024;
/// Persisted lifecycle of a logical call, independent of its current transport.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RecoveredSessionStatus {
Open,
Ending,
Closed,
}
pub(crate) enum HandoffSettlement {
Settled,
Wait(std::sync::Arc<tokio::sync::Notify>),
}
// One transport utterance can span several frames when a promotion or typed input
// seals speech midway through it. V3 drains complete utterances in capture order.
pub(super) struct V3Utterance {
role: RealtimeTranscriptRole,
pub(super) frames: Vec<String>,
pub(super) complete: bool,
}
#[derive(Clone, Debug)]
pub(crate) struct PreparedRealtimeInput {
pub(crate) id: Uuid,
pub(crate) session_id: String,
pub(crate) handoff_id: String,
pub(crate) sources: Vec<String>,
consumed_event_ids: Vec<String>,
pub(crate) input: String,
pub(crate) new_items: Vec<RealtimeItem>,
pub(crate) reserved: bool,
reserved_turn_id: Option<String>,
pub(crate) cached_envelope: Option<ResponseItemEnvelope>,
}
impl RealtimeHistoryState {
/// Recover only durable V3 speech. A new transport starts fresh utterances; frames from
/// before reconnect remain frozen in persisted order until the next input consumes them.
pub(crate) fn recover_session(
&mut self,
session_id: &str,
items: &[RealtimeItem],
) -> RecoveredSessionStatus {
let mut consumed = std::collections::HashSet::new();
let mut status = RecoveredSessionStatus::Open;
for item in items {
if let Some(time) = item.observed_at_ms {
self.observation_clock.observe(time);
}
match &item.content {
RealtimeItemContent::TurnInputAccepted {
handoff_id,
source_event_ids,
..
} => {
self.committed_handoff_ids.insert(handoff_id.clone());
consumed.extend(source_event_ids.iter().cloned());
}
RealtimeItemContent::SessionInputSaved {
reason,
source_event_ids,
..
} => {
consumed.extend(source_event_ids.iter().cloned());
if *reason == SessionInputReason::SessionEnd {
status = RecoveredSessionStatus::Ending;
}
}
RealtimeItemContent::RealtimeSessionClosed { .. } => {
status = RecoveredSessionStatus::Closed
}
_ => {}
}
}
self.active_session_id =
(status != RecoveredSessionStatus::Closed).then(|| session_id.to_owned());
self.preserve_transcript = true;
self.active_segments = Default::default();
self.v3_utterances.clear();
self.pending_handoff_seals.clear();
self.pending_transcript = items
.iter()
.filter(|item| {
matches!(item.content, RealtimeItemContent::Frame { .. })
&& !consumed.contains(&item.id)
})
.cloned()
.collect();
self.recovered_frame_ids = self
.pending_transcript
.iter()
.map(|item| item.id.clone())
.collect();
status
}
/// Build one chunk's exact frame provenance before recording its canonical model input.
/// Frozen speech must precede new speech so reconnect cannot reverse model input order.
pub(crate) fn prepare_recorded_chunk(
&self,
entries: &[RealtimeTranscriptEntry],
) -> (Vec<RealtimeTranscriptEntry>, Vec<String>) {
let mut combined = Vec::new();
let mut sources = Vec::new();
for item in &self.pending_transcript {
if self.recovered_frame_ids.contains(&item.id)
&& let RealtimeItemContent::Frame { channels } = &item.content
{
let transcript = &channels.voice_transcript;
combined.push(RealtimeTranscriptEntry {
role: match transcript.role {
RealtimeTranscriptRole::User => "user",
RealtimeTranscriptRole::Assistant => "assistant",
}
.to_owned(),
text: transcript.text.clone(),
});
sources.push(item.id.clone());
}
}
sources.extend(
self.v3_utterances
.iter()
.take(entries.len())
.take_while(|u| u.complete)
.flat_map(|u| u.frames.iter().cloned()),
);
combined.extend_from_slice(entries);
(combined, sources)
}
pub(crate) fn active_session_id(&self) -> Option<&str> {
self.active_session_id.as_deref()
}
pub(crate) fn reserved_turn_ids(&self) -> Vec<String> {
self.handoff_inputs
.values()
.filter(|input| input.reserved)
.filter_map(|input| input.reserved_turn_id.clone())
.collect()
}
pub(super) fn remember_transcript(&mut self, item: RealtimeItem) {
self.pending_transcript.push_back(item);
if self.preserve_transcript {
return;
}
let mut bytes: usize = self.pending_transcript.iter().map(transcript_bytes).sum();
while bytes > PENDING_TRANSCRIPT_BYTES && self.pending_transcript.len() > 1 {
if let Some(item) = self.pending_transcript.pop_front() {
bytes -= transcript_bytes(&item);
}
}
}
pub(super) fn v3_utterance(
&mut self,
role: RealtimeTranscriptRole,
start: bool,
) -> Option<&mut V3Utterance> {
if !self.frames_enabled() || !self.preserve_transcript {
return None;
}
let utterances = &mut self.v3_utterances;
let index = match utterances
.iter()
.rposition(|utterance| !utterance.complete && utterance.role == role)
{
Some(index) => index,
None if start => {
utterances.push_back(V3Utterance {
role,
frames: Vec::new(),
complete: false,
});
utterances.len() - 1
}
None => return None,
};
utterances.get_mut(index)
}
pub(crate) fn consume_recorded_chunk(&mut self, entries: &[RealtimeTranscriptEntry]) {
// The socket drains only the complete prefix. Match by utterance position,
// not text: a late final can repeat speech already sealed at a handoff.
let count = self
.v3_utterances
.iter()
.take(entries.len())
.take_while(|u| u.complete)
.count();
let mut ids: std::collections::HashSet<_> = self
.v3_utterances
.drain(..count)
.flat_map(|u| u.frames)
.collect();
ids.extend(self.recovered_frame_ids.drain());
self.pending_transcript
.retain(|item| !ids.contains(&item.id));
}
/// Seal active speech and leave frames reserved by an in-flight input out of this one.
fn seal_available_transcripts(&mut self) -> Vec<(Option<&str>, RealtimeTranscriptRole, &str)> {
let mut new_items = Vec::new();
self.seal_segments(&mut new_items, Continuation::Continue);
self.pending_handoff_seals.extend(new_items);
if !self.v3_utterances.is_empty() {
// Speakers can finish out of order, and a promotion can split an utterance.
// Keep each utterance's frames together in original speech order.
let rank: std::collections::HashMap<_, _> = self
.v3_utterances
.iter()
.flat_map(|utterance| &utterance.frames)
.enumerate()
.map(|(index, id)| (id, index + 1))
.collect();
self.pending_transcript
.make_contiguous()
.sort_by_key(|item| rank.get(&item.id).copied().unwrap_or(0));
}
let reserved = self
.handoff_inputs
.values()
.filter(|input| input.reserved || input.cached_envelope.is_some())
.flat_map(|input| input.consumed_event_ids.iter())
.collect::<std::collections::HashSet<_>>();
self.pending_transcript
.iter()
.filter(|item| !reserved.contains(&item.id))
.filter_map(|item| {
if let RealtimeItemContent::Frame { channels } = &item.content {
let transcript = &channels.voice_transcript;
Some((
Some(item.id.as_str()),
transcript.role,
transcript.text.as_str(),
))
} else {
None
}
})
.collect()
}
/// A pending frame belongs to only one immutable seal candidate. Rejection leaves it available.
fn unassigned_handoff_seals(&self) -> Vec<RealtimeItem> {
let assigned = self
.handoff_inputs
.values()
.flat_map(|input| input.new_items.iter().map(|item| &item.id))
.collect::<std::collections::HashSet<_>>();
self.pending_handoff_seals
.iter()
.filter(|item| !assigned.contains(&item.id))
.cloned()
.collect()
}
pub(crate) fn prepare_handoff(
&mut self,
handoff: &RealtimeHandoffRequested,
) -> Option<PreparedRealtimeInput> {
if self.committed_handoff_ids.contains(&handoff.handoff_id) {
return None;
}
if let Some(prepared) = self.handoff_inputs.get(&handoff.handoff_id) {
return Some(prepared.clone());
}
let session_id = self.active_session_id.clone()?;
let preserve_transcript = self.preserve_transcript;
// The V3 transport appends a tool input row only if it did not already
// hear that utterance, including in a chunk it previously recorded.
let has_tool_input_row = handoff.active_transcript.len() > self.v3_utterances.len();
let mut entries = self.seal_available_transcripts();
let consumed_event_ids = entries
.iter()
.filter_map(|(id, _, _)| id.map(str::to_string))
.collect();
let fallback = handoff.input_transcript.trim();
let add_fallback = if preserve_transcript {
has_tool_input_row
} else {
!entries.iter().any(|(_, role, text)| {
*role == RealtimeTranscriptRole::User && text.trim() == fallback
})
};
if !fallback.is_empty() && add_fallback {
// Tool-supplied task text is context, not evidence that the user spoke it.
// Preserve it in the accepted input without inventing a transcript event.
entries.push((None, RealtimeTranscriptRole::User, fallback));
}
let (input, sources) = RealtimeDelegation::from_events(
&handoff.input_transcript,
&entries,
RealtimeDelegationSource::Handoff,
preserve_transcript,
)?;
let prepared = PreparedRealtimeInput {
id: Uuid::now_v7(),
session_id,
handoff_id: handoff.handoff_id.clone(),
sources,
consumed_event_ids,
input,
new_items: self.unassigned_handoff_seals(),
reserved: false,
reserved_turn_id: None,
cached_envelope: None,
};
self.handoff_inputs
.insert(handoff.handoff_id.clone(), prepared.clone());
Some(prepared)
}
pub(crate) fn restore_handoffs(&mut self, items: &[codex_history::RolloutItem]) {
for handoff_id in items
.iter()
.filter_map(codex_history::RolloutItem::realtime_handoff_id)
{
self.committed_handoff_ids.insert(handoff_id.to_string());
}
}
/// Reserve before queuing the input so overlapping handoffs cannot claim its frames.
pub(crate) fn reserve_handoff(&mut self, id: Uuid) -> bool {
let Some(input) = self
.handoff_inputs
.values_mut()
.find(|input| input.id == id)
else {
return false;
};
if input.reserved {
return false;
}
input.reserved = true;
true
}
pub(crate) fn pending_handoff(&self, id: Uuid) -> Option<PreparedRealtimeInput> {
self.handoff_inputs
.values()
.find(|input| input.id == id)
.cloned()
}
/// Keep the exact annotated input across an uncertain append or flush outcome.
pub(crate) fn cache_handoff_input(
&mut self,
id: Uuid,
envelope: ResponseItemEnvelope,
) -> Option<PreparedRealtimeInput> {
let input = self
.handoff_inputs
.values_mut()
.find(|input| input.id == id)?;
input.cached_envelope.get_or_insert(envelope);
Some(input.clone())
}
/// A definite rejection frees its sources; an uncertain seal retains the exact candidate.
pub(crate) fn reject_handoff(&mut self, id: Uuid) {
let Some((key, input)) = self
.handoff_inputs
.iter_mut()
.find(|(_, input)| input.id == id)
else {
return;
};
if input.cached_envelope.is_some() {
input.reserved = false;
input.reserved_turn_id = None;
} else {
let key = key.clone();
self.handoff_inputs.remove(&key);
}
self.handoff_settled.notify_one();
}
pub(crate) fn bind_handoff_to_turn(&mut self, id: Uuid, turn_id: &str) {
if let Some(input) = self
.handoff_inputs
.values_mut()
.find(|input| input.id == id)
&& input.reserved
{
input.reserved_turn_id = Some(turn_id.to_owned());
}
}
pub(crate) fn reject_handoffs_for_turn(&mut self, turn_id: &str) {
let ids = self
.handoff_inputs
.values()
.filter(|input| input.reserved_turn_id.as_deref() == Some(turn_id))
.map(|input| input.id)
.collect::<Vec<_>>();
for id in ids {
self.reject_handoff(id);
}
}
/// Subscribe while the state is locked; notify_one retains a permit if settlement races await.
pub(crate) fn pending_handoff_settlement(&self) -> std::io::Result<HandoffSettlement> {
if self.handoff_inputs.values().any(|input| input.reserved) {
return Ok(HandoffSettlement::Wait(std::sync::Arc::clone(
&self.handoff_settled,
)));
}
if self
.handoff_inputs
.values()
.any(|input| input.cached_envelope.is_some())
{
return Err(std::io::Error::other(
"realtime handoff seal outcome is uncertain",
));
}
Ok(HandoffSettlement::Settled)
}
/// Consume transcript only after the canonical input (or hangup receipt) is durable.
pub(crate) fn finish_handoff(&mut self, id: Uuid, turn_id: Option<&str>) {
let Some(handoff_id) = self
.handoff_inputs
.iter()
.find_map(|(key, input)| (input.id == id).then(|| key.clone()))
else {
return;
};
let Some(prepared) = self.handoff_inputs.remove(&handoff_id) else {
return;
};
self.pending_handoff_seals
.retain(|item| !prepared.new_items.iter().any(|sealed| sealed.id == item.id));
self.pending_transcript
.retain(|item| !prepared.consumed_event_ids.contains(&item.id));
self.recovered_frame_ids
.retain(|id| !prepared.consumed_event_ids.contains(id));
if let Some(turn_id) = turn_id {
self.realtime_session_by_bem_turn
.insert(turn_id.to_string(), prepared.session_id);
}
self.committed_handoff_ids.insert(handoff_id);
if prepared.reserved {
self.handoff_settled.notify_one();
}
}
}
fn transcript_bytes(item: &RealtimeItem) -> usize {
match &item.content {
RealtimeItemContent::Frame { channels } => channels.voice_transcript.text.len(),
_ => 0,
}
}
+144
View File
@@ -619,6 +619,73 @@ fn promotes_successful_dynamic_tools_and_splits_active_transcripts() {
assert_eq!(resumed[0].realtime_session_id, "voice-2");
}
#[test]
fn overlapping_committed_handoffs_seal_each_frame_once_and_rejection_releases_it() {
use codex_protocol::protocol::RealtimeHandoffRequested;
let mut state = started_state();
state.frame_enrollment = Some(true);
let request = |id: &str| RealtimeHandoffRequested {
handoff_id: id.to_owned(),
item_id: id.to_owned(),
input_transcript: String::new(),
active_transcript: Vec::new(),
};
let first_frame = observe_realtime(
&mut state,
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
delta: "first".to_owned(),
}),
)
.transcript_stream
.unwrap()
.item_id;
let first = state.prepare_handoff(&request("first")).unwrap();
assert!(state.reserve_handoff(first.id));
let second_frame = observe_realtime(
&mut state,
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
delta: " second".to_owned(),
}),
)
.transcript_stream
.unwrap()
.item_id;
let second = state.prepare_handoff(&request("second")).unwrap();
assert!(state.reserve_handoff(second.id));
assert_eq!(
first
.new_items
.iter()
.map(|item| &item.id)
.collect::<Vec<_>>(),
vec![&first_frame]
);
assert_eq!(
second
.new_items
.iter()
.map(|item| &item.id)
.collect::<Vec<_>>(),
vec![&second_frame]
);
state.finish_handoff(first.id, Some("turn-first"));
state.finish_handoff(second.id, Some("turn-second"));
observe_realtime(
&mut state,
RealtimeEvent::InputTranscriptDelta(RealtimeTranscriptDelta {
delta: " third".to_owned(),
}),
);
let rejected = state.prepare_handoff(&request("rejected")).unwrap();
assert!(state.reserve_handoff(rejected.id));
state.reject_handoff(rejected.id);
let retry = state.prepare_handoff(&request("retry")).unwrap();
assert_eq!(retry.new_items, rejected.new_items);
}
#[test_case(|item| EventMsg::ItemStarted(ItemStartedEvent {
thread_id: ThreadId::new(),
turn_id: "turn-1".to_string(),
@@ -682,3 +749,80 @@ fn typed_input_seals_both_roles_but_realtime_delegation_does_not(
assert!(observe_realtime(&mut state, event).items.is_empty());
}
}
#[test]
fn recovered_v3_speech_excludes_exact_consumed_frames_and_precedes_new_chunks() {
use super::handoff::RecoveredSessionStatus;
use codex_protocol::protocol::RealtimeTranscriptEntry;
use codex_protocol::realtime::SessionInputReason;
let item = |id: &str, content| RealtimeItem {
id: id.to_owned(),
realtime_session_id: "call".to_owned(),
observed_at_ms: Some(1),
content,
};
let frame = |id: &str, role, text: &str| {
item(
id,
RealtimeItemContent::Frame {
channels: RealtimeFrameChannels {
voice_transcript: RealtimeVoiceTranscript {
role,
text: text.to_owned(),
},
},
},
)
};
let items = vec![
item("start", RealtimeItemContent::RealtimeSessionStarted),
frame("a1", RealtimeTranscriptRole::User, "first"),
frame("b1", RealtimeTranscriptRole::Assistant, "pending"),
frame("a2", RealtimeTranscriptRole::User, "last"),
item(
"chunk",
RealtimeItemContent::SessionInputSaved {
reason: SessionInputReason::Chunk,
source_event_ids: vec!["a1".to_owned(), "a2".to_owned()],
input_item_id: "input".to_owned(),
input: "first last".to_owned(),
},
),
];
let mut state = RealtimeHistoryState::default();
assert_eq!(
state.recover_session("call", &items),
RecoveredSessionStatus::Open
);
let new = vec![RealtimeTranscriptEntry {
role: "user".to_owned(),
text: "new".to_owned(),
}];
assert_eq!(
state.prepare_recorded_chunk(&new),
(
vec![
RealtimeTranscriptEntry {
role: "assistant".to_owned(),
text: "pending".to_owned()
},
new[0].clone()
],
vec!["b1".to_owned()]
)
);
let mut ending = items;
ending.push(item(
"end",
RealtimeItemContent::SessionInputSaved {
reason: SessionInputReason::SessionEnd,
source_event_ids: vec!["b1".to_owned()],
input_item_id: "tail".to_owned(),
input: "pending".to_owned(),
},
));
assert_eq!(
state.recover_session("call", &ending),
RecoveredSessionStatus::Ending
);
}
+3
View File
@@ -30,6 +30,9 @@ pub struct UserInputMetadata {
/// Optional host-assigned UUID for this particular Responses user item.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub response_item_uuid: Option<uuid::Uuid>,
/// Core-assigned handoff identity carried through the input queue.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub realtime_handoff_id: Option<uuid::Uuid>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_dictation: Option<bool>,
#[serde(
+138 -44
View File
@@ -278,6 +278,7 @@ mod model_catalog_settings;
mod model_ids;
pub(crate) mod multi_agents;
mod plugin_selection;
#[expect(dead_code, reason = "Activated with hangup in the next PR")]
mod realtime_handoff;
mod realtime_history;
mod retained_context;
@@ -1984,11 +1985,9 @@ impl Session {
rollout_items: &[RolloutItem],
) -> Option<PreviousTurnSettings> {
if let Some(history) = &self.realtime_history {
history
.lock()
.await
.observation_clock
.restore(rollout_items);
let mut history = history.lock().await;
history.observation_clock.restore(rollout_items);
history.restore_handoffs(rollout_items);
}
let rollout_reconstruction::RolloutReconstruction {
mut history,
@@ -4084,9 +4083,33 @@ impl Session {
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
mut items: Vec<ResponseItemEnvelope>,
items: Vec<ResponseItemEnvelope>,
image_preparations: Vec<ImagePreparationMetadata>,
) {
// Ordinary history persistence retains its existing best-effort behavior.
let _ = self
.record_prepared_items(
turn_context,
model_info,
items,
image_preparations,
/*realtime*/ None,
)
.await;
}
#[expect(
clippy::await_holding_invalid_type,
reason = "a durable realtime admission must publish history atomically with respect to other history writers"
)]
async fn record_prepared_items(
&self,
turn_context: &TurnContext,
model_info: &ModelInfo,
mut items: Vec<ResponseItemEnvelope>,
image_preparations: Vec<ImagePreparationMetadata>,
realtime: Option<crate::realtime_history::handoff::PreparedRealtimeInput>,
) -> std::io::Result<()> {
// Save the originating history budget for replay.
// Preserve any existing tool-specific override.
let policy: codex_utils_output_truncation::TruncationPolicy =
@@ -4113,10 +4136,16 @@ impl Session {
let force_mcp_checkpoint = items
.iter()
.any(|envelope| crate::context_manager::is_user_turn_boundary(&envelope.item));
let mcp_revision = self
.services
.executed_tool_calls
.mcp_attribution_checkpoint(force_mcp_checkpoint)
let cached_retry = realtime
.as_ref()
.is_some_and(|input| input.cached_envelope.is_some());
let mcp_revision = (!cached_retry)
.then(|| {
self.services
.executed_tool_calls
.mcp_attribution_checkpoint(force_mcp_checkpoint)
})
.flatten()
.and_then(|(attribution, revision)| {
let first_persisted = items
.iter()
@@ -4133,15 +4162,30 @@ impl Session {
}
Some(revision)
});
let response_items = items
// Thread settings precede the first accepted input. Keep the persistence lock
// through its seal, and clear the pending flag only after the seal succeeds.
let realtime_settings_guard = if realtime.is_some()
&& force_mcp_checkpoint
&& self.reference_context_item().await.is_none()
{
Some(thread_settings::acquire_persistence_lock(self).await)
} else {
None
};
let realtime_initial_settings = realtime_settings_guard.is_some()
&& self.reference_context_item().await.is_none()
&& self.state.lock().await.initial_settings_pending();
let realtime_settings_event = if realtime_initial_settings {
Some(thread_settings::applied_event(self).await)
} else {
None
};
let mut response_items = items
.iter()
.map(|envelope| envelope.item.clone())
.collect::<Vec<_>>();
{
let mut state = self.state.lock().await;
state
.current_time_reminder
.note_recorded_items(&response_items);
let pending_orders = turn_context
.extension_data
.get::<retained_context::PendingAssistantMessageOrders>();
@@ -4175,12 +4219,41 @@ impl Session {
});
}
}
if let Some(prepared) = &realtime {
let envelope = if let Some(cached) = &prepared.cached_envelope {
cached.clone()
} else {
let [envelope] = items.as_slice() else {
return Err(std::io::Error::other(
"realtime admission must contain one input",
));
};
envelope.clone()
};
let envelope = self
.record_realtime_handoff_input(
&mut state,
&turn_context.sub_id,
envelope,
prepared,
realtime_settings_event,
policy,
)
.await?;
if realtime_initial_settings {
state.take_initial_settings_pending();
}
response_items = vec![envelope.item.clone()];
items = vec![envelope];
} else {
state
.pending_audio_duration_ids
.extend(file_audio::file_ids(&items).cloned());
state.history.record_annotated_items(&mut items, policy);
}
state
.pending_audio_duration_ids
.extend(file_audio::file_ids(&items).cloned());
state
.history
.record_annotated_items(&mut items, model_info.truncation_policy.into());
.current_time_reminder
.note_recorded_items(&response_items);
}
for image in image_preparations {
self.services
@@ -4190,27 +4263,32 @@ impl Session {
metadata: image,
});
}
let settings_guard =
if force_mcp_checkpoint && self.reference_context_item().await.is_none() {
Some(thread_settings::acquire_persistence_lock(self).await)
let persisted = if let Some(prepared) = &realtime {
self.finish_realtime_handoff(prepared, &items[0]).await;
true
} else {
let settings_guard =
if force_mcp_checkpoint && self.reference_context_item().await.is_none() {
Some(thread_settings::acquire_persistence_lock(self).await)
} else {
None
};
let initial_settings = if settings_guard.is_some()
&& self.reference_context_item().await.is_none()
&& self.state.lock().await.take_initial_settings_pending()
{
Some(thread_settings::applied_event(self).await)
} else {
None
};
let initial_settings = if settings_guard.is_some()
&& self.reference_context_item().await.is_none()
&& self.state.lock().await.take_initial_settings_pending()
{
Some(thread_settings::applied_event(self).await)
} else {
None
let rollout_items = initial_settings
.into_iter()
.map(RolloutItem::EventMsg)
.chain(items.into_iter().map(RolloutItem::ResponseItem))
.collect::<Vec<_>>();
self.persist_rollout_items(&rollout_items).await
};
let rollout_items = initial_settings
.into_iter()
.map(RolloutItem::EventMsg)
.chain(items.into_iter().map(RolloutItem::ResponseItem))
.collect::<Vec<_>>();
let persisted = self.persist_rollout_items(&rollout_items).await;
drop(settings_guard);
drop(realtime_settings_guard);
if persisted && let Some(revision) = mcp_revision {
self.services
.executed_tool_calls
@@ -4225,6 +4303,7 @@ impl Session {
}
self.send_raw_response_items(turn_context, &response_items)
.await;
Ok(())
}
pub(crate) async fn record_step_world_state_if_changed(
@@ -6006,7 +6085,11 @@ impl Session {
client_id: Option<String>,
metadata: UserInputMetadata,
persist_context: PersistContext,
) {
) -> std::io::Result<()> {
let realtime = match metadata.realtime_handoff_id {
Some(id) => self.pending_realtime_handoff(id, input).await,
None => None,
};
let display_input = input
.iter()
.map(ResolvedUserInput::to_user_input)
@@ -6049,18 +6132,28 @@ impl Session {
&prepared_items,
&user_image_content_indices,
);
self.record_prepared_conversation_items(
turn_context,
model_info,
prepared_items,
image_preparations,
)
.await;
if let Err(error) = self
.record_prepared_items(
turn_context,
model_info,
prepared_items,
image_preparations,
realtime.clone(),
)
.await
{
if let Some(prepared) = realtime {
self.reject_realtime_handoff(prepared.id).await;
}
return Err(error);
}
if metadata.origin.is_user() {
self.services
.analytics_events_client
.track_accepted_user_input(&turn_context.sub_id, metadata.used_dictation);
}
// Existing clients use this accepted input item to identify delegated turns.
// The realtime receipt adds provenance without changing that live contract.
user_message_item.client_id = client_id;
let turn_item = TurnItem::UserMessage(user_message_item);
self.emit_turn_item_started(turn_context, &turn_item).await;
@@ -6072,6 +6165,7 @@ impl Session {
)
.await;
self.ensure_rollout_materialized(persist_context).await;
Ok(())
}
pub(crate) async fn notify_stream_error(
+367 -11
View File
@@ -1,21 +1,227 @@
//! Routes realtime input through ordinary turn admission.
//! Routes realtime input through ordinary admission and publishes its durable history projection.
use super::session::Session;
use super::turn_input;
use crate::realtime_conversation::RealtimeConversationManager;
use crate::realtime_history::handoff::HandoffSettlement;
use crate::realtime_history::handoff::PreparedRealtimeInput;
use crate::state::SessionState;
use codex_history::RealtimeHandoffMetadata;
use codex_history::ResponseItemEnvelope;
use codex_history::RolloutItem;
use codex_protocol::protocol::CodexErrorInfo;
use codex_protocol::protocol::ErrorEvent;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::RealtimeConversationRealtimeEvent;
use codex_protocol::protocol::RealtimeEvent;
use codex_protocol::realtime::RealtimeHandoffMode;
use codex_protocol::realtime::RealtimeItem;
use codex_protocol::realtime::RealtimeItemContent;
use codex_protocol::turn_input::NotSubmittedReason;
use codex_protocol::turn_input::TurnInputMode;
use codex_protocol::turn_input::TurnInputRequest;
use codex_protocol::turn_input::TurnInputSubmission;
use codex_protocol::turn_input::TurnStartOptions;
use codex_user_input::ResolvedUserInput;
use codex_utils_output_truncation::TruncationPolicy;
use std::sync::Arc;
use uuid::Uuid;
/// A canceled seal has an unknown durable outcome and cannot leave the live thread usable.
struct RealtimeSealGuard {
conversation: Option<Arc<RealtimeConversationManager>>,
}
impl RealtimeSealGuard {
fn new(conversation: Arc<RealtimeConversationManager>) -> Self {
Self {
conversation: Some(conversation),
}
}
fn disarm(&mut self) {
self.conversation = None;
}
}
impl Drop for RealtimeSealGuard {
fn drop(&mut self) {
if let Some(conversation) = self.conversation.take() {
conversation.mark_history_recovery_required();
drop(tokio::spawn(async move {
conversation.stop_for_history_recovery().await;
}));
}
}
}
impl Session {
pub(super) async fn record_realtime_handoff_input(
&self,
state: &mut SessionState,
turn_id: &str,
mut envelope: ResponseItemEnvelope,
prepared: &PreparedRealtimeInput,
settings_event: Option<EventMsg>,
policy: TruncationPolicy,
) -> std::io::Result<ResponseItemEnvelope> {
if self.conversation.history_recovery_required() {
return Err(std::io::Error::other(
"realtime history recovery is required",
));
}
let replay = prepared.cached_envelope.is_some();
if !replay {
envelope.metadata.get_or_insert_default().realtime_handoff = Some(Box::new(
self.realtime_handoff_metadata(prepared, turn_id).await?,
));
}
if envelope.item.turn_id() != Some(turn_id) || envelope.realtime_handoff_item().is_none() {
return Err(std::io::Error::other("invalid realtime accepted input"));
}
// Reserve sequence and order without adding model-facing content. A failed
// seal may leave a gap, but siblings cannot reuse an uncertain input's ID.
state.history.prepare_item_sampling(&mut envelope);
let staged_retained_context = state.history.stage_realtime_input(&mut envelope);
let live = self
.live_thread_for_persistence("accept realtime handoff")
.map_err(std::io::Error::other)?;
if let Some(event) = settings_event {
live.append_items(&[RolloutItem::EventMsg(event)])
.await
.map_err(std::io::Error::other)?;
}
let prepared = self
.cache_realtime_handoff(prepared.id, envelope.clone())
.await?;
let mut seal_guard = RealtimeSealGuard::new(Arc::clone(&self.conversation));
// Confirm an uncertain write before another input can advance history. The store
// contract makes this retry idempotent even when the first write actually landed.
if let Err(first_error) = live
.seal_realtime_input(prepared.new_items.clone(), envelope.clone())
.await
&& let Err(error) = live
.seal_realtime_input(prepared.new_items.clone(), envelope.clone())
.await
{
tracing::warn!(%first_error, %error, "could not confirm realtime input seal");
self.conversation.mark_history_recovery_required();
seal_guard.disarm();
return Err(std::io::Error::other(error));
}
// Fresh recording exposes the source only after capturing the review backup;
// a cached retry replays its already persisted provenance.
let fresh_source = if replay {
None
} else {
envelope
.metadata
.as_mut()
.and_then(|metadata| metadata.retained_source.take())
};
state
.history
.record_sealed_realtime_input(&envelope, staged_retained_context, policy);
seal_guard.disarm();
if let Some(source) = fresh_source {
envelope.metadata.get_or_insert_default().retained_source = Some(source);
}
Ok(envelope)
}
pub(crate) async fn route_realtime_handoff(
self: &Arc<Self>,
handoff: &codex_protocol::protocol::RealtimeHandoffRequested,
preserve_transcript: bool,
) -> Result<(), &'static str> {
let history = match &self.realtime_history {
Some(history) if history.lock().await.frames_enabled() => history,
_ => {
if let Some(text) = crate::realtime_conversation::realtime_delegation_from_handoff(
handoff,
preserve_transcript,
) {
return self.route_realtime_text_input(text).await;
}
return Ok(());
}
};
let prepared = {
let mut state = history.lock().await;
let Some(prepared) = state.prepare_handoff(handoff) else {
return Ok(());
};
if !state.reserve_handoff(prepared.id) {
return Ok(());
}
prepared
};
let submission_id = prepared.id.to_string();
let mode = if let Some(envelope) = &prepared.cached_envelope {
let Some(target) = envelope.item.turn_id() else {
self.reject_realtime_handoff(prepared.id).await;
return Err("realtime handoff is missing its receiving turn");
};
if self.turn_context_for_sub_id(target).await.is_some() {
TurnInputMode::Steer {
expected_turn_id: target.to_string(),
}
} else if target == submission_id
&& envelope
.metadata
.as_ref()
.and_then(|metadata| metadata.realtime_handoff.as_ref())
.is_some_and(|metadata| metadata.mode == RealtimeHandoffMode::Start)
{
TurnInputMode::StartIfIdle
} else {
self.reject_realtime_handoff(prepared.id).await;
return Err("resume the receiving turn before retrying this realtime handoff");
}
} else {
TurnInputMode::StartOrSteer
};
let mut request =
TurnInputRequest::user_input(vec![ResolvedUserInput::text(prepared.input.clone())])
.on_start(TurnStartOptions {
turn_trigger: Some("realtime".to_string()),
..Default::default()
});
request.user_item_uuid = Some(prepared.id);
match turn_input::handle_realtime_handoff(
self,
request,
mode,
submission_id.clone(),
prepared.id,
)
.await
{
Ok(TurnInputSubmission::Accepted) => {
unreachable!("realtime user input returns a turn outcome")
}
Ok(TurnInputSubmission::Started { .. } | TurnInputSubmission::Steered { .. }) => Ok(()),
Ok(TurnInputSubmission::NotSubmitted {
reason: NotSubmittedReason::ServerDraining,
}) => {
self.reject_realtime_handoff(prepared.id).await;
Err("Server is draining; retry the turn after reconnecting")
}
Ok(TurnInputSubmission::NotSubmitted { reason }) => {
self.reject_realtime_handoff(prepared.id).await;
self.report_realtime_input_decline(submission_id, reason)
.await;
Ok(())
}
Err(error) => {
self.reject_realtime_handoff(prepared.id).await;
tracing::warn!("failed to admit realtime handoff: {error}");
Err("failed to admit realtime handoff")
}
}
}
pub(crate) async fn route_realtime_text_input(
self: &Arc<Self>,
text: String,
@@ -44,16 +250,8 @@ impl Session {
return Err("Server is draining; retry the turn after reconnecting");
}
Ok(TurnInputSubmission::NotSubmitted { reason }) => {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(ErrorEvent {
misalignment: None,
message: format!("failed to submit turn input: {reason:?}"),
codex_error_info: Some(CodexErrorInfo::BadRequest),
rejected_audio_file_ids: Vec::new(),
}),
})
.await;
self.report_realtime_input_decline(submission_id, reason)
.await;
}
Err(error) => {
self.send_event_raw(Event {
@@ -65,4 +263,162 @@ impl Session {
}
Ok(())
}
async fn report_realtime_input_decline(
&self,
submission_id: String,
reason: NotSubmittedReason,
) {
self.send_event_raw(Event {
id: submission_id,
msg: EventMsg::Error(ErrorEvent {
misalignment: None,
message: format!("failed to submit turn input: {reason:?}"),
codex_error_info: Some(CodexErrorInfo::BadRequest),
rejected_audio_file_ids: Vec::new(),
}),
})
.await;
}
/// UUID and exact text must both match a reserved internal input. Client UUIDs alone
/// do not attach realtime provenance to an ordinary prompt.
pub(crate) async fn pending_realtime_handoff(
&self,
id: Uuid,
input: &[ResolvedUserInput],
) -> Option<PreparedRealtimeInput> {
let [ResolvedUserInput::Text { text, .. }] = input else {
return None;
};
let prepared = self
.realtime_history
.as_ref()?
.lock()
.await
.pending_handoff(id)?;
(prepared.reserved && prepared.input == *text).then_some(prepared)
}
pub(crate) async fn realtime_handoff_metadata(
&self,
prepared: &PreparedRealtimeInput,
turn_id: &str,
) -> std::io::Result<RealtimeHandoffMetadata> {
let history = self
.realtime_history
.as_ref()
.ok_or_else(|| std::io::Error::other("realtime history is disabled"))?;
let observed_at_ms = history.lock().await.observation_clock.now_ms();
Ok(RealtimeHandoffMetadata {
event_id: prepared.id.to_string(),
realtime_session_id: prepared.session_id.clone(),
observed_at_ms,
handoff_id: prepared.handoff_id.clone(),
mode: if turn_id == prepared.id.to_string() {
RealtimeHandoffMode::Start
} else {
RealtimeHandoffMode::Steer
},
source_event_ids: prepared.sources.clone(),
})
}
pub(crate) async fn cache_realtime_handoff(
&self,
id: Uuid,
envelope: ResponseItemEnvelope,
) -> std::io::Result<PreparedRealtimeInput> {
self.realtime_history
.as_ref()
.ok_or_else(|| std::io::Error::other("realtime history is disabled"))?
.lock()
.await
.cache_handoff_input(id, envelope)
.ok_or_else(|| std::io::Error::other("realtime handoff is no longer pending"))
}
pub(crate) async fn finish_realtime_handoff(
&self,
prepared: &PreparedRealtimeInput,
envelope: &ResponseItemEnvelope,
) {
if let Some(history) = &self.realtime_history {
history
.lock()
.await
.finish_handoff(prepared.id, envelope.item.turn_id());
}
self.publish_realtime_acceptance(&prepared.new_items).await;
if let Some(receipt) = envelope.realtime_handoff_item() {
self.publish_realtime_acceptance(std::slice::from_ref(&receipt))
.await;
}
}
pub(crate) async fn bind_realtime_handoffs_to_turn(
&self,
input: &[super::TurnInput],
turn_id: &str,
) {
let ids = input
.iter()
.filter_map(|input| match input {
super::TurnInput::UserInput { metadata, .. } => metadata.realtime_handoff_id,
_ => None,
})
.collect::<Vec<_>>();
if ids.is_empty() {
return;
}
if let Some(history) = &self.realtime_history {
let mut state = history.lock().await;
for id in ids {
state.bind_handoff_to_turn(id, turn_id);
}
}
}
pub(crate) async fn reject_realtime_handoffs_for_turn(&self, turn_id: &str) {
if let Some(history) = &self.realtime_history {
history.lock().await.reject_handoffs_for_turn(turn_id);
}
}
/// Wait for admitted inputs to be durable or rejected, not for their agent turns to finish.
pub(crate) async fn wait_for_realtime_handoffs(&self) -> std::io::Result<()> {
let Some(history) = &self.realtime_history else {
return Ok(());
};
loop {
let pending = history.lock().await.pending_handoff_settlement();
match pending? {
HandoffSettlement::Wait(settled) => settled.notified().await,
HandoffSettlement::Settled => return Ok(()),
}
}
}
pub(crate) async fn reject_realtime_handoff(&self, id: Uuid) {
if let Some(history) = &self.realtime_history {
history.lock().await.reject_handoff(id);
}
}
async fn publish_realtime_acceptance(&self, records: &[RealtimeItem]) {
for item in records {
let event = |payload| Event {
id: item.id.clone(),
msg: EventMsg::RealtimeConversationRealtime(RealtimeConversationRealtimeEvent {
payload,
}),
};
if !matches!(item.content, RealtimeItemContent::Frame { .. }) {
self.deliver_event_raw(event(RealtimeEvent::HistoryItemStarted(item.clone())))
.await;
}
self.deliver_event_raw(event(RealtimeEvent::HistoryItemCompleted(item.clone())))
.await;
}
}
}
+21 -6
View File
@@ -1161,17 +1161,21 @@ pub(crate) async fn run_hooks_and_record_inputs(
) -> CodexResult<bool> {
let mut blocked_input = false;
let mut accepted_user_input = false;
let mut input_error = None;
for input_item in recordable_inputs(sess, model_info, input) {
let input_item = input_item?;
let hook_outcome = inspect_pending_input(sess, turn_context, &input_item).await;
let realtime_handoff_id = match input_item.as_ref() {
TurnInput::UserInput { metadata, .. } => metadata.realtime_handoff_id,
_ => None,
};
if hook_outcome.should_stop {
if let Some(id) = realtime_handoff_id {
sess.reject_realtime_handoff(id).await;
}
blocked_input = true;
record_additional_contexts(sess, turn_context, hook_outcome.additional_contexts).await;
} else {
if matches!(input_item.as_ref(), TurnInput::UserInput { content, .. } if !content.is_empty())
{
accepted_user_input = true;
}
// Tool outputs retain their durability barrier, including in mixed input batches.
let input_persist_context = if persist_context == PersistContext::SteeredUserInput
&& matches!(input_item.as_ref(), TurnInput::FunctionCallOutput(_))
@@ -1180,7 +1184,8 @@ pub(crate) async fn run_hooks_and_record_inputs(
} else {
persist_context
};
record_pending_input(
let is_user_input = matches!(input_item.as_ref(), TurnInput::UserInput { content, .. } if !content.is_empty());
if let Err(error) = record_pending_input(
sess,
turn_context,
model_info,
@@ -1188,7 +1193,13 @@ pub(crate) async fn run_hooks_and_record_inputs(
hook_outcome.additional_contexts,
input_persist_context,
)
.await;
.await
{
// The batch has left the queue. Persist its unrelated siblings before failing.
input_error.get_or_insert(error);
} else if is_user_input {
accepted_user_input = true;
}
}
}
// The batch has left the input queue. Persist every accepted item before the
@@ -1196,6 +1207,10 @@ pub(crate) async fn run_hooks_and_record_inputs(
if !input.is_empty() {
super::file_audio::resolve_pending(sess).await;
}
if let Some(error) = input_error {
sess.conversation.stop_for_history_recovery().await;
return Err(CodexErr::Io(error));
}
Ok(blocked_input && !accepted_user_input)
}
+70 -14
View File
@@ -243,19 +243,53 @@ impl PreparedTurnInputSettings {
}
}
#[tracing::instrument(
name = "codex.turn_input",
level = "trace",
skip_all,
fields(conversation.id = %session.thread_id, turn.id)
)]
pub(super) async fn handle(
session: &Arc<Session>,
request: impl Into<WithTurnExtensionData<TurnInputRequest>>,
mode: TurnInputMode,
submission_id: String,
) -> CodexResult<TurnInputSubmission> {
let request = request.into();
handle_inner(
session,
request.into(),
mode,
submission_id,
/*realtime_handoff_id*/ None,
)
.await
}
pub(super) async fn handle_realtime_handoff(
session: &Arc<Session>,
request: TurnInputRequest,
mode: TurnInputMode,
submission_id: String,
handoff_id: uuid::Uuid,
) -> CodexResult<TurnInputSubmission> {
handle_inner(
session,
request.into(),
mode,
submission_id,
Some(handoff_id),
)
.await
}
#[tracing::instrument(
name = "codex.turn_input",
level = "trace",
skip_all,
fields(conversation.id = %session.thread_id, turn.id)
)]
async fn handle_inner(
session: &Arc<Session>,
request: WithTurnExtensionData<TurnInputRequest>,
mode: TurnInputMode,
submission_id: String,
realtime_handoff_id: Option<uuid::Uuid>,
) -> CodexResult<TurnInputSubmission> {
session.conversation.check_history_recovery()?;
if request.request.start.service_tier.is_some()
&& crate::agent::host::host_child_uses_shared_service_tier(session)
{
@@ -285,7 +319,9 @@ pub(super) async fn handle(
.map_err(|message| CodexErr::InvalidRequest(message.to_owned()))?;
}
let result = match mode {
TurnInputMode::StartOrSteer => start_or_steer(session, request, submission_id).await,
TurnInputMode::StartOrSteer => {
start_or_steer(session, request, submission_id, realtime_handoff_id).await
}
TurnInputMode::StartIfIdle => {
let kind = match &request.request.input {
SubmittedTurnInput::UserInput { content, .. } if !content.is_empty() => {
@@ -302,6 +338,7 @@ pub(super) async fn handle(
submission_id,
kind,
/*expected_previous_turn_id*/ None,
realtime_handoff_id,
)
.await
}
@@ -329,11 +366,19 @@ pub(super) async fn handle(
submission_id,
TurnStartKind::Recovery,
Some(expected_previous_turn_id),
realtime_handoff_id,
)
.await
}
TurnInputMode::Steer { expected_turn_id } => {
steer(session, request, expected_turn_id, submission_id).await
steer(
session,
request,
expected_turn_id,
submission_id,
realtime_handoff_id,
)
.await
}
};
// Link this request's trace to the accepted turn, which may have an older trace.
@@ -358,6 +403,7 @@ pub(super) async fn handle_recovery(
start_options: TurnStartOptions,
submission_id: String,
) -> CodexResult<TurnInputSubmission> {
session.conversation.check_history_recovery()?;
let WithTurnExtensionData {
request: thread_settings,
turn_extension_init,
@@ -374,6 +420,7 @@ pub(super) async fn handle_recovery(
submission_id,
TurnStartKind::Recovery,
/*expected_previous_turn_id*/ None,
/*realtime_handoff_id*/ None,
)
.await;
if let Ok(TurnInputSubmission::Started { turn_id, .. }) = &result {
@@ -382,7 +429,10 @@ pub(super) async fn handle_recovery(
result
}
fn user_input_metadata(request: &mut TurnInputRequest) -> UserInputMetadata {
fn user_input_metadata(
request: &mut TurnInputRequest,
realtime_handoff_id: Option<uuid::Uuid>,
) -> UserInputMetadata {
const DICTATION_KEY: &str = "codex_app_used_dictation";
// Capture provenance on this input before applying mutable turn metadata.
let metadata = &mut request.responsesapi_client_metadata;
@@ -398,15 +448,17 @@ fn user_input_metadata(request: &mut TurnInputRequest) -> UserInputMetadata {
used_dictation: patch.or(replacement).and_then(|value| value.parse().ok()),
origin: UserInputOrigin::from_turn_trigger(request.start.turn_trigger.as_deref()),
response_item_uuid: request.user_item_uuid,
realtime_handoff_id,
extension_data: request.user_input_extension_data.clone(),
..Default::default()
}
}
async fn start_or_steer(
pub(super) async fn start_or_steer(
session: &Arc<Session>,
request: WithTurnExtensionData<TurnInputRequest>,
submission_id: String,
realtime_handoff_id: Option<uuid::Uuid>,
) -> CodexResult<TurnInputSubmission> {
if matches!(
request.request.input,
@@ -418,7 +470,7 @@ async fn start_or_steer(
mut request,
turn_extension_init,
} = request;
let input_metadata = user_input_metadata(&mut request);
let input_metadata = user_input_metadata(&mut request, realtime_handoff_id);
let TurnInputRequest {
mut input,
thread_settings,
@@ -540,12 +592,13 @@ async fn start_if_idle(
submission_id: String,
kind: TurnStartKind,
expected_previous_turn_id: Option<String>,
realtime_handoff_id: Option<uuid::Uuid>,
) -> CodexResult<TurnInputSubmission> {
let WithTurnExtensionData {
mut request,
turn_extension_init,
} = request;
let input_metadata = user_input_metadata(&mut request);
let input_metadata = user_input_metadata(&mut request, realtime_handoff_id);
let TurnInputRequest {
mut input,
thread_settings,
@@ -719,12 +772,13 @@ async fn steer(
request: WithTurnExtensionData<TurnInputRequest>,
expected_turn_id: String,
submission_id: String,
realtime_handoff_id: Option<uuid::Uuid>,
) -> CodexResult<TurnInputSubmission> {
let WithTurnExtensionData {
mut request,
turn_extension_init,
} = request;
let input_metadata = user_input_metadata(&mut request);
let input_metadata = user_input_metadata(&mut request, realtime_handoff_id);
let TurnInputRequest {
mut input,
thread_settings,
@@ -1045,6 +1099,8 @@ impl Session {
}
};
pending_input.push(input);
self.bind_realtime_handoffs_to_turn(&pending_input, active_turn_id)
.await;
self.input_queue
.extend_pending_input_and_notify_for_turn_state(
active_turn.turn_state.as_ref(),
@@ -1159,7 +1159,7 @@ fn dictation_metadata_is_captured_per_input_and_removed_from_responses() {
"false".to_owned(),
)])));
assert_eq!(
user_input_metadata(&mut request).used_dictation,
user_input_metadata(&mut request, /*realtime_handoff_id*/ None).used_dictation,
Some(false)
);
assert_eq!(
@@ -1227,7 +1227,8 @@ async fn scored_prompt_keeps_its_submission_item_id() {
metadata,
codex_thread_store::PersistContext::Standard,
)
.await;
.await
.expect("record user input");
}
let history = session.clone_history().await;
let ids: Vec<_> = history
+69
View File
@@ -1,14 +1,83 @@
use super::*;
use crate::realtime_history::RealtimeHistoryState;
use crate::session::UserInputMetadata;
use crate::session::tests::make_session_and_context;
use codex_extension_api::ExtensionData;
use codex_extension_api::TurnItemContributor;
use codex_protocol::ResponseItemId;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::ItemStartedEvent;
use codex_protocol::protocol::RealtimeConversationStartedEvent;
use codex_protocol::protocol::RealtimeConversationVersion;
use codex_protocol::protocol::RealtimeHandoffRequested;
use pretty_assertions::assert_eq;
use std::sync::Arc;
use tracing_subscriber::prelude::*;
#[tokio::test]
async fn failed_realtime_receipt_preserves_sibling_input_before_sampling() {
// The fixture has no live thread, so committing a reserved handoff fails.
let (mut session, turn_context) = make_session_and_context().await;
let mut history = RealtimeHistoryState::default();
history.frame_enrollment = Some(true);
history.observe(&EventMsg::RealtimeConversationStarted(
RealtimeConversationStartedEvent {
realtime_session_id: Some("call".to_string()),
version: RealtimeConversationVersion::V2,
},
));
let handoff = history
.prepare_handoff(&RealtimeHandoffRequested {
handoff_id: "handoff".to_string(),
item_id: "item".to_string(),
input_transcript: "voice prompt".to_string(),
active_transcript: Vec::new(),
})
.unwrap();
assert!(history.reserve_handoff(handoff.id));
session.realtime_history = Some(tokio::sync::Mutex::new(history));
let sibling_id = uuid::Uuid::now_v7();
let input = |text, id, realtime_handoff_id| TurnInput::UserInput {
content: vec![ResolvedUserInput::text(text)],
client_id: None,
metadata: UserInputMetadata {
response_item_uuid: Some(id),
realtime_handoff_id,
..Default::default()
},
};
let session = Arc::new(session);
let error = run_turn(
Arc::clone(&session),
Arc::new(turn_context),
vec![
input(handoff.input, handoff.id, Some(handoff.id)),
input(
"typed sibling".to_string(),
sibling_id,
/*realtime_handoff_id*/ None,
),
],
&mut McpStartupRequirements::default(),
/*prewarmed_client_session*/ None,
CancellationToken::new(),
)
.await
.expect_err("a turn with an unconfirmed handoff must fail before sampling");
assert!(error.to_string().contains("accept realtime handoff"));
let sibling = ResponseItemId::for_admitted_user_input(sibling_id);
let voice = ResponseItemId::for_admitted_user_input(handoff.id);
let history = session.clone_history().await;
let recorded: Vec<_> = history
.raw_items()
.filter_map(ResponseItem::id)
.filter(|id| *id == &sibling || *id == &voice)
.collect();
assert_eq!(recorded, vec![&sibling]);
}
struct RewriteAgentMessageContributor;
impl TurnItemContributor for RewriteAgentMessageContributor {
+4
View File
@@ -208,6 +208,10 @@ impl SessionState {
pending
}
pub(crate) fn initial_settings_pending(&self) -> bool {
self.initial_settings_pending
}
pub(crate) fn clone_history(&self) -> ContextManager {
self.history.clone()
}
+15 -2
View File
@@ -105,6 +105,7 @@ impl Session {
return Ok(());
}
let mut input_error = None;
for item in recordable_inputs(self, &model_info, &input) {
let item = item?;
// The user must resubmit a prompt that still needs a submit hook.
@@ -117,9 +118,14 @@ impl Session {
.execution_mode_for_event(HookEventName::UserPromptSubmit)
.is_some()
{
if let TurnInput::UserInput { metadata, .. } = item.as_ref()
&& let Some(uuid) = metadata.realtime_handoff_id
{
self.reject_realtime_handoff(uuid).await;
}
continue;
}
record_pending_input(
if let Err(error) = record_pending_input(
self,
context,
&model_info,
@@ -127,7 +133,14 @@ impl Session {
Vec::new(),
PersistContext::Standard,
)
.await;
.await
{
input_error.get_or_insert(error);
}
}
if let Some(error) = input_error {
self.conversation.stop_for_history_recovery().await;
return Err(CodexErr::Io(error));
}
Ok(())
}
+7
View File
@@ -416,6 +416,8 @@ impl Session {
(task_kind == TaskKind::Regular).then(|| turn_context.attribution()),
)
.await;
self.bind_realtime_handoffs_to_turn(&input, &turn_context.sub_id)
.await;
let host_admission = self.host_turn_admission();
let (agent_execution_guard, task_input, pending_initial_input) = match &host_admission {
Some(_) => (None, Vec::new(), Some(input)),
@@ -867,6 +869,8 @@ impl Session {
PersistContext::Standard,
)
.await;
self.reject_realtime_handoffs_for_turn(&turn_context.sub_id)
.await;
if let Some(task) = unadmitted_task {
// The task never ran. Unwind setup such as entering review mode.
task.abort(Arc::clone(self), Arc::clone(&turn_context))
@@ -1258,6 +1262,9 @@ impl Session {
.await;
}
self.reject_realtime_handoffs_for_turn(&task.turn_context.sub_id)
.await;
session_task
.abort(Arc::clone(self), Arc::clone(&task.turn_context))
.await;
+26 -7
View File
@@ -278,11 +278,10 @@ impl CodexHarnessMetadata {
}
impl ResponseItemEnvelope {
/// Projects the realtime view of an accepted input without storing a second copy.
pub fn realtime_handoff_item(&self) -> Option<RealtimeItem> {
let metadata = self.metadata.as_ref()?.realtime_handoff.as_ref()?;
fn validated_realtime_handoff(&self) -> Option<(&RealtimeHandoffMetadata, &str)> {
let metadata = self.metadata.as_ref()?.realtime_handoff.as_deref()?;
let ResponseItem::Message {
id: Some(id),
id: Some(_),
role,
content,
..
@@ -293,9 +292,15 @@ impl ResponseItemEnvelope {
let [ContentItem::InputText { text }] = content.as_slice() else {
return None;
};
if role != "user" {
if role != "user" || self.item.turn_id().is_none() {
return None;
}
Some((metadata, text))
}
/// Projects the realtime view of an accepted input without storing a second copy.
pub fn realtime_handoff_item(&self) -> Option<RealtimeItem> {
let (metadata, text) = self.validated_realtime_handoff()?;
Some(RealtimeItem {
id: metadata.event_id.clone(),
realtime_session_id: metadata.realtime_session_id.clone(),
@@ -305,8 +310,8 @@ impl ResponseItemEnvelope {
turn_id: self.item.turn_id()?.to_string(),
mode: metadata.mode,
source_event_ids: metadata.source_event_ids.clone(),
input_item_id: id.to_string(),
input: text.clone(),
input_item_id: self.item.id()?.to_string(),
input: text.to_string(),
},
})
}
@@ -428,6 +433,20 @@ pub enum RolloutItem {
}
impl RolloutItem {
/// Returns the handoff ID only when the rollout item is a valid acceptance receipt.
pub fn realtime_handoff_id(&self) -> Option<&str> {
match self {
Self::RealtimeItem(RealtimeItem {
content: RealtimeItemContent::TurnInputAccepted { handoff_id, .. },
..
}) => Some(handoff_id),
Self::ResponseItem(envelope) => envelope
.validated_realtime_handoff()
.map(|(metadata, _)| metadata.handoff_id.as_str()),
_ => None,
}
}
/// Returns the canonical realtime observation time without constructing a projection.
pub fn realtime_observed_at_ms(&self) -> Option<i64> {
match self {