Derive realtime input receipts from thread history (#62251)

## Why

Accepted handoffs and saved transcript chunks need to appear in realtime
history. Their text can come from the same user message that model history
uses, avoiding a second persisted copy for the receipt.

## What changed

- Reconstruct `turnInputAccepted` and transcript chunk receipts from user
  messages annotated with realtime provenance. Use the stored observation
  time when restoring the realtime clock.
- Replace `sessionEndInputSaved` in the app-server protocol with
  `sessionInputSaved` and a `reason` of `chunk` or `sessionEnd`. Update the
  generated schemas and bindings.

## Testing

Add coverage for reconstructing both receipt types, preserving rollout
order in paginated realtime history, and restoring observation time
across a restart after wall-clock rollback.

GitOrigin-RevId: fcb4e0e539f4e4066726e83fa1e945fb659a2ca8
This commit is contained in:
Zahan Malkani
2026-11-18 11:01:02 +00:00
committed by copyberry
parent 6d9f0199ed
commit 4389a526d5
19 changed files with 404 additions and 65 deletions
@@ -6341,6 +6341,9 @@
"input_item_id": {
"type": "string"
},
"reason": {
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
},
"source_event_ids": {
"items": {
"type": "string"
@@ -6349,19 +6352,20 @@
},
"type": {
"enum": [
"sessionEndInputSaved"
"sessionInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"title": "SessionInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"reason",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"title": "SessionInputSavedThreadRealtimeItem",
"type": "object"
}
],
@@ -6485,6 +6489,13 @@
],
"type": "object"
},
"ThreadRealtimeSessionInputReason": {
"enum": [
"chunk",
"sessionEnd"
],
"type": "string"
},
"ThreadRealtimeSessionOutcome": {
"enum": [
"ended",
@@ -25871,6 +25871,9 @@
"input_item_id": {
"type": "string"
},
"reason": {
"$ref": "#/definitions/v2/ThreadRealtimeSessionInputReason"
},
"source_event_ids": {
"items": {
"type": "string"
@@ -25879,19 +25882,20 @@
},
"type": {
"enum": [
"sessionEndInputSaved"
"sessionInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"title": "SessionInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"reason",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"title": "SessionInputSavedThreadRealtimeItem",
"type": "object"
}
],
@@ -26045,6 +26049,13 @@
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeSessionInputReason": {
"enum": [
"chunk",
"sessionEnd"
],
"type": "string"
},
"ThreadRealtimeSessionOutcome": {
"enum": [
"ended",
@@ -23567,6 +23567,9 @@
"input_item_id": {
"type": "string"
},
"reason": {
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
},
"source_event_ids": {
"items": {
"type": "string"
@@ -23575,19 +23578,20 @@
},
"type": {
"enum": [
"sessionEndInputSaved"
"sessionInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"title": "SessionInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"reason",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"title": "SessionInputSavedThreadRealtimeItem",
"type": "object"
}
],
@@ -23741,6 +23745,13 @@
"title": "ThreadRealtimeSdpNotification",
"type": "object"
},
"ThreadRealtimeSessionInputReason": {
"enum": [
"chunk",
"sessionEnd"
],
"type": "string"
},
"ThreadRealtimeSessionOutcome": {
"enum": [
"ended",
@@ -241,6 +241,9 @@
"input_item_id": {
"type": "string"
},
"reason": {
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
},
"source_event_ids": {
"items": {
"type": "string"
@@ -249,19 +252,20 @@
},
"type": {
"enum": [
"sessionEndInputSaved"
"sessionInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"title": "SessionInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"reason",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"title": "SessionInputSavedThreadRealtimeItem",
"type": "object"
}
],
@@ -287,6 +291,13 @@
],
"type": "object"
},
"ThreadRealtimeSessionInputReason": {
"enum": [
"chunk",
"sessionEnd"
],
"type": "string"
},
"ThreadRealtimeSessionOutcome": {
"enum": [
"ended",
@@ -241,6 +241,9 @@
"input_item_id": {
"type": "string"
},
"reason": {
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
},
"source_event_ids": {
"items": {
"type": "string"
@@ -249,19 +252,20 @@
},
"type": {
"enum": [
"sessionEndInputSaved"
"sessionInputSaved"
],
"title": "SessionEndInputSavedThreadRealtimeItemType",
"title": "SessionInputSavedThreadRealtimeItemType",
"type": "string"
}
},
"required": [
"input",
"input_item_id",
"reason",
"source_event_ids",
"type"
],
"title": "SessionEndInputSavedThreadRealtimeItem",
"title": "SessionInputSavedThreadRealtimeItem",
"type": "object"
}
],
@@ -287,6 +291,13 @@
],
"type": "object"
},
"ThreadRealtimeSessionInputReason": {
"enum": [
"chunk",
"sessionEnd"
],
"type": "string"
},
"ThreadRealtimeSessionOutcome": {
"enum": [
"ended",
@@ -4,6 +4,7 @@
import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
import type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
import type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
import type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason";
import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
@@ -14,4 +15,4 @@ export type ThreadRealtimeItem = { id: string, realtimeSessionId: string,
/**
* Canonical observation time in Unix milliseconds for eligible voice calls.
*/
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionEndInputSaved", sourceEventIds: Array<string>, inputItemId: string, input: string, });
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionInputSaved", reason: ThreadRealtimeSessionInputReason, sourceEventIds: Array<string>, inputItemId: string, input: string, });
@@ -0,0 +1,5 @@
// GENERATED CODE! DO NOT MODIFY BY HAND!
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
export type ThreadRealtimeSessionInputReason = "chunk" | "sessionEnd";
@@ -599,6 +599,7 @@ export type { ThreadRealtimeItemTranscriptDeltaNotification } from "./ThreadReal
export type { ThreadRealtimeOrder } from "./ThreadRealtimeOrder";
export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification";
export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification";
export type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason";
export type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport";
export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification";
@@ -13,6 +13,7 @@ use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole;
use codex_protocol::realtime::SessionInputReason as CoreSessionInputReason;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value as JsonValue;
@@ -66,7 +67,8 @@ pub enum ThreadRealtimeItemContent {
input_item_id: String,
input: String,
},
SessionEndInputSaved {
SessionInputSaved {
reason: ThreadRealtimeSessionInputReason,
source_event_ids: Vec<String>,
input_item_id: String,
input: String,
@@ -81,6 +83,14 @@ pub enum ThreadRealtimeHandoffMode {
Steer,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(rename_all = "camelCase", export_to = "v2/")]
pub enum ThreadRealtimeSessionInputReason {
Chunk,
SessionEnd,
}
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(
@@ -225,11 +235,18 @@ impl ThreadRealtimeItem {
input_item_id,
input,
},
CoreRealtimeItemContent::SessionEndInputSaved {
CoreRealtimeItemContent::SessionInputSaved {
reason,
source_event_ids,
input_item_id,
input,
} => ThreadRealtimeItemContent::SessionEndInputSaved {
} => ThreadRealtimeItemContent::SessionInputSaved {
reason: match reason {
CoreSessionInputReason::Chunk => ThreadRealtimeSessionInputReason::Chunk,
CoreSessionInputReason::SessionEnd => {
ThreadRealtimeSessionInputReason::SessionEnd
}
},
source_event_ids,
input_item_id,
input,
@@ -485,7 +485,7 @@ pub(crate) async fn apply_bespoke_event_handling(
if matches!(
&item.content,
codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. }
| codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. }
| codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. }
) => {}
RealtimeEvent::HistoryItemStarted(item) => {
outgoing
+7 -1
View File
@@ -28,6 +28,12 @@ fn observation_time_survives_clock_rollback_and_owner_restart() {
observed_at_ms: Some(110),
content: RealtimeItemContent::RealtimeSessionStarted,
}),
serde_json::from_value(serde_json::json!({
"type": "response_item",
"payload": {"type": "message", "id": "msg_input_1", "role": "user", "content": [{"type": "input_text", "text": "hello"}]},
"metadata": {"realtime_handoff": {"event_id": "receipt", "realtime_session_id": "voice", "observed_at_ms": 115, "handoff_id": "handoff", "mode": "start", "source_event_ids": []}}
}))
.expect("read annotated handoff input"),
RolloutItem::RealtimeItem(RealtimeItem {
id: "legacy".into(),
realtime_session_id: "legacy-voice".into(),
@@ -40,7 +46,7 @@ fn observation_time_survives_clock_rollback_and_owner_restart() {
clock.observe(/*wall_clock_ms*/ 80),
clock.observe(/*wall_clock_ms*/ 120)
],
[110, 120]
[115, 120]
);
}
+113
View File
@@ -45,7 +45,10 @@ use codex_protocol::protocol::ThreadSource;
use codex_protocol::protocol::TokenUsageRecord;
use codex_protocol::protocol::TurnContextItem;
use codex_protocol::protocol::WorldStateItem;
use codex_protocol::realtime::RealtimeHandoffMode;
use codex_protocol::realtime::RealtimeItem;
use codex_protocol::realtime::RealtimeItemContent;
use codex_protocol::realtime::SessionInputReason;
use codex_protocol::security_risk::SecurityRiskScore;
use codex_protocol::turn_input::InjectedResponseItem;
use codex_protocol::turn_input::ItemRetention;
@@ -81,6 +84,14 @@ pub enum ModelSearchState {
///
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
pub struct CodexHarnessMetadata {
/// Realtime provenance for an accepted ordinary turn input. Never model-visible.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub realtime_handoff: Option<Box<RealtimeHandoffMetadata>>,
/// Provenance of a transcript chunk saved in ordinary model history.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub realtime_session_input: Option<Box<RealtimeSessionInputMetadata>>,
/// Thread-local append sequence. Compaction keeps the identities of retained inputs.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub history_sequence: Option<u64>,
@@ -200,6 +211,26 @@ pub struct CodexHarnessMetadata {
pub sender_user_messages: Option<Box<SenderUserMessages>>,
}
/// Facts that connect an ordinary accepted input to its originating realtime frames.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
pub struct RealtimeHandoffMetadata {
pub event_id: String,
pub realtime_session_id: String,
pub observed_at_ms: i64,
pub handoff_id: String,
pub mode: RealtimeHandoffMode,
pub source_event_ids: Vec<String>,
}
/// Realtime provenance for a saved transcript chunk; the response item owns its text.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
pub struct RealtimeSessionInputMetadata {
pub event_id: String,
pub realtime_session_id: String,
pub observed_at_ms: i64,
pub source_event_ids: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, JsonSchema)]
pub struct ImageReplaySource {
pub prepared_url_digest: Vec<u8>,
@@ -247,6 +278,70 @@ impl CodexHarnessMetadata {
}
impl ResponseItemEnvelope {
/// Projects the realtime view of an accepted input without storing a second copy.
pub fn realtime_handoff_item(&self) -> Option<RealtimeItem> {
let metadata = self.metadata.as_ref()?.realtime_handoff.as_ref()?;
let ResponseItem::Message {
id: Some(id),
role,
content,
..
} = &self.item
else {
return None;
};
let [ContentItem::InputText { text }] = content.as_slice() else {
return None;
};
if role != "user" {
return None;
}
Some(RealtimeItem {
id: metadata.event_id.clone(),
realtime_session_id: metadata.realtime_session_id.clone(),
observed_at_ms: Some(metadata.observed_at_ms),
content: RealtimeItemContent::TurnInputAccepted {
handoff_id: metadata.handoff_id.clone(),
turn_id: self.item.turn_id()?.to_string(),
mode: metadata.mode,
source_event_ids: metadata.source_event_ids.clone(),
input_item_id: id.to_string(),
input: text.clone(),
},
})
}
/// Projects a chunk receipt from its canonical model input.
pub fn realtime_session_input_item(&self) -> Option<RealtimeItem> {
let metadata = self.metadata.as_ref()?.realtime_session_input.as_ref()?;
let ResponseItem::Message {
id: Some(id),
role,
content,
..
} = &self.item
else {
return None;
};
let [ContentItem::InputText { text }] = content.as_slice() else {
return None;
};
if role != "user" {
return None;
}
Some(RealtimeItem {
id: metadata.event_id.clone(),
realtime_session_id: metadata.realtime_session_id.clone(),
observed_at_ms: Some(metadata.observed_at_ms),
content: RealtimeItemContent::SessionInputSaved {
reason: SessionInputReason::Chunk,
source_event_ids: metadata.source_event_ids.clone(),
input_item_id: id.to_string(),
input: text.clone(),
},
})
}
/// Whether the client requested that this item survive context-window changes and forks.
pub fn is_always_retained(&self) -> bool {
self.metadata
@@ -337,6 +432,20 @@ impl RolloutItem {
pub fn realtime_observed_at_ms(&self) -> Option<i64> {
match self {
Self::RealtimeItem(item) => item.observed_at_ms,
Self::ResponseItem(envelope) => envelope
.metadata
.as_ref()?
.realtime_handoff
.as_ref()
.map(|metadata| metadata.observed_at_ms)
.or_else(|| {
envelope
.metadata
.as_ref()?
.realtime_session_input
.as_ref()
.map(|metadata| metadata.observed_at_ms)
}),
_ => None,
}
}
@@ -345,6 +454,10 @@ impl RolloutItem {
pub fn realtime_item(&self) -> Option<Cow<'_, RealtimeItem>> {
match self {
Self::RealtimeItem(item) => Some(Cow::Borrowed(item)),
Self::ResponseItem(envelope) => envelope
.realtime_handoff_item()
.or_else(|| envelope.realtime_session_input_item())
.map(Cow::Owned),
_ => None,
}
}
+92
View File
@@ -16,6 +16,98 @@ use serde_json::json;
use super::*;
#[test]
fn accepted_realtime_input_projects_from_the_canonical_response_item() -> Result<()> {
let mut item: ResponseItem = serde_json::from_value(json!({
"type": "message",
"id": "msg_input_123",
"role": "user",
"content": [{"type": "input_text", "text": "use the smaller logo"}]
}))?;
item.set_turn_id_if_missing("turn-1");
let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope {
item,
metadata: Some(CodexHarnessMetadata {
realtime_handoff: Some(Box::new(RealtimeHandoffMetadata {
event_id: "receipt-1".to_string(),
realtime_session_id: "call-1".to_string(),
observed_at_ms: 42,
handoff_id: "handoff-1".to_string(),
mode: RealtimeHandoffMode::Start,
source_event_ids: vec!["frame-1".to_string()],
})),
..Default::default()
}),
});
let persisted = serde_json::to_value(&rollout)?;
assert_eq!(persisted["type"], "response_item");
assert_eq!(
persisted["payload"]["content"][0]["text"],
"use the smaller logo"
);
assert_eq!(
persisted["metadata"]["realtime_handoff"]["event_id"],
"receipt-1"
);
let restored: RolloutItem = serde_json::from_value(persisted)?;
assert_eq!(
restored.realtime_item().map(Cow::into_owned),
Some(RealtimeItem {
id: "receipt-1".to_string(),
realtime_session_id: "call-1".to_string(),
observed_at_ms: Some(42),
content: RealtimeItemContent::TurnInputAccepted {
handoff_id: "handoff-1".to_string(),
turn_id: "turn-1".to_string(),
mode: RealtimeHandoffMode::Start,
source_event_ids: vec!["frame-1".to_string()],
input_item_id: "msg_input_123".to_string(),
input: "use the smaller logo".to_string(),
},
})
);
Ok(())
}
#[test]
fn transcript_chunk_projects_from_its_only_model_input() -> Result<()> {
let mut item: ResponseItem = serde_json::from_value(json!({
"type": "message", "id": "msg_chunk", "role": "user",
"content": [{"type": "input_text", "text": "user: hello"}]
}))?;
item.set_turn_id_if_missing("turn-1");
let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope {
item,
metadata: Some(CodexHarnessMetadata {
realtime_session_input: Some(Box::new(RealtimeSessionInputMetadata {
event_id: "receipt-1".to_string(),
realtime_session_id: "call-1".to_string(),
observed_at_ms: 42,
source_event_ids: vec!["frame-1".to_string()],
})),
..Default::default()
}),
});
let persisted = serde_json::to_value(&rollout)?;
assert_eq!(persisted["type"], "response_item");
assert_eq!(persisted["payload"]["content"][0]["text"], "user: hello");
assert_eq!(
rollout.realtime_item().map(Cow::into_owned),
Some(RealtimeItem {
id: "receipt-1".to_string(),
realtime_session_id: "call-1".to_string(),
observed_at_ms: Some(42),
content: RealtimeItemContent::SessionInputSaved {
reason: SessionInputReason::Chunk,
source_event_ids: vec!["frame-1".to_string()],
input_item_id: "msg_chunk".to_string(),
input: "user: hello".to_string(),
},
})
);
Ok(())
}
fn thread_settings_snapshot(disabled_plugin_ids: Vec<String>) -> Result<ThreadSettingsSnapshot> {
Ok(serde_json::from_value(json!({
"model": "gpt-5",
+11 -2
View File
@@ -63,14 +63,23 @@ pub enum RealtimeItemContent {
input_item_id: String,
input: String,
},
/// End-of-call input recorded in model history without starting or steering a turn.
SessionEndInputSaved {
/// Transcript input saved in model history without starting or steering an agent turn.
SessionInputSaved {
reason: SessionInputReason,
source_event_ids: Vec<String>,
input_item_id: String,
input: String,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
#[ts(rename_all = "snake_case")]
pub enum SessionInputReason {
Chunk,
SessionEnd,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)]
#[serde(rename_all = "snake_case")]
#[ts(rename_all = "snake_case")]
+1 -1
View File
@@ -285,7 +285,7 @@ fn rollout_item_type(item: &RolloutItem) -> String {
codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. } => {
"realtime.turn_input_accepted".to_string()
}
codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. } => {
codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. } => {
"realtime.session_end_input_saved".to_string()
}
},
@@ -8,6 +8,7 @@ use std::time::Duration;
use chrono::Utc;
use codex_app_server_protocol::McpToolCallResult;
use codex_app_server_protocol::ThreadItemPayload;
use codex_protocol::ResponseItemId;
use codex_protocol::ThreadId;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::AgentMessageItem;
@@ -41,6 +42,7 @@ use codex_protocol::realtime::RealtimeItem;
use codex_protocol::realtime::RealtimeItemContent;
use codex_protocol::realtime::RealtimeSessionOutcome;
use codex_protocol::realtime::RealtimeTranscriptRole;
use codex_rollout::ResponseItemEnvelope;
use codex_rollout::RolloutConfig;
use codex_rollout::RolloutItem;
use codex_rollout::RolloutLine;
@@ -619,6 +621,35 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
.await
.expect("persist thread metadata");
let mut handoff_input = ResponseItem::Message {
encrypted_content: None,
status: None,
id: Some(ResponseItemId::from_server("msg_input_handoff".to_string())),
role: "user".to_string(),
content: vec![ContentItem::InputText {
text: "use the smaller logo".to_string(),
}],
phase: None,
internal_chat_message_metadata_passthrough: None,
};
handoff_input.set_turn_id_if_missing("turn-1");
let handoff_input = RolloutItem::ResponseItem(ResponseItemEnvelope {
item: handoff_input,
metadata: Some(
serde_json::from_value(serde_json::json!({
"realtime_handoff": {
"event_id": "voice:handoff",
"realtime_session_id": "voice",
"observed_at_ms": 42,
"handoff_id": "handoff",
"mode": "start",
"source_event_ids": ["voice:transcript:0"]
}
}))
.expect("deserialize handoff provenance"),
),
});
store
.append_items(AppendThreadItemsParams {
thread_id,
@@ -639,6 +670,7 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
},
}),
turn_started("turn-1"),
handoff_input,
RolloutItem::RealtimeItem(RealtimeItem {
id: "voice:promoted:agent-1".to_string(),
realtime_session_id: "voice".to_string(),
@@ -685,22 +717,23 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
.expect("read canonical rollout")
.lines()
.map(|line| codex_rollout::parse_rollout_line(line).expect("parse rollout line"))
.filter_map(|line| match line.item {
RolloutItem::RealtimeItem(item) => Some((
item.id,
i64::try_from(line.ordinal.expect("paginated rollout ordinal"))
.expect("ordinal fits SQLite integer"),
chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str())
.expect("valid rollout timestamp")
.timestamp_millis(),
serde_json::to_value(item.content)
.expect("serialize realtime content")
.get("type")
.and_then(serde_json::Value::as_str)
.expect("realtime content discriminator")
.to_string(),
)),
_ => None,
.filter_map(|line| {
line.item.realtime_item().map(|item| {
(
item.id.clone(),
i64::try_from(line.ordinal.expect("paginated rollout ordinal"))
.expect("ordinal fits SQLite integer"),
chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str())
.expect("valid rollout timestamp")
.timestamp_millis(),
serde_json::to_value(&item.content)
.expect("serialize realtime content")
.get("type")
.and_then(serde_json::Value::as_str)
.expect("realtime content discriminator")
.to_string(),
)
})
})
.collect::<Vec<_>>();
assert_eq!(
@@ -711,8 +744,9 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
vec![
("voice:started", 1, "realtime_session_started"),
("voice:transcript:0", 2, "transcript_segment"),
("voice:promoted:agent-1", 4, "bem_item_promoted"),
("voice:closed", 6, "realtime_session_closed"),
("voice:handoff", 4, "turn_input_accepted"),
("voice:promoted:agent-1", 5, "bem_item_promoted"),
("voice:closed", 7, "realtime_session_closed"),
]
);
+29 -23
View File
@@ -6520,27 +6520,6 @@ class TurnInputAcceptedThreadRealtimeItem(BaseModel):
]
class SessionEndInputSavedThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
input: str
input_item_id: str
source_event_ids: list[str]
type: Annotated[
Literal["sessionEndInputSaved"], Field(title="SessionEndInputSavedThreadRealtimeItemType")
]
class ThreadRealtimeItemAddedNotification(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
@@ -6579,6 +6558,11 @@ class ThreadRealtimeSdpNotification(BaseModel):
thread_id: Annotated[str, Field(alias="threadId")]
class ThreadRealtimeSessionInputReason(Enum):
chunk = "chunk"
session_end = "sessionEnd"
class ThreadRealtimeSessionOutcome(Enum):
ended = "ended"
failed = "failed"
@@ -11405,6 +11389,28 @@ class RealtimeSessionClosedThreadRealtimeItem(BaseModel):
]
class SessionInputSavedThreadRealtimeItem(BaseModel):
model_config = ConfigDict(
populate_by_name=True,
)
id: str
observed_at_ms: Annotated[
int | None,
Field(
alias="observedAtMs",
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
),
] = None
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
input: str
input_item_id: str
reason: ThreadRealtimeSessionInputReason
source_event_ids: list[str]
type: Annotated[
Literal["sessionInputSaved"], Field(title="SessionInputSavedThreadRealtimeItemType")
]
class ThreadRealtimeItem(
RootModel[
RealtimeSessionStartedThreadRealtimeItem
@@ -11413,7 +11419,7 @@ class ThreadRealtimeItem(
| BemItemPromotedThreadRealtimeItem
| RealtimeSessionClosedThreadRealtimeItem
| TurnInputAcceptedThreadRealtimeItem
| SessionEndInputSavedThreadRealtimeItem
| SessionInputSavedThreadRealtimeItem
]
):
root: Annotated[
@@ -11423,7 +11429,7 @@ class ThreadRealtimeItem(
| BemItemPromotedThreadRealtimeItem
| RealtimeSessionClosedThreadRealtimeItem
| TurnInputAcceptedThreadRealtimeItem
| SessionEndInputSavedThreadRealtimeItem,
| SessionInputSavedThreadRealtimeItem,
Field(
description="EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline."
),