mirror of
https://github.com/openai/codex.git
synced 2026-11-18 22:20:48 +00:00
Derive realtime input receipts from thread history (#62251)
## Why Accepted handoffs and saved transcript chunks need to appear in realtime history. Their text can come from the same user message that model history uses, avoiding a second persisted copy for the receipt. ## What changed - Reconstruct `turnInputAccepted` and transcript chunk receipts from user messages annotated with realtime provenance. Use the stored observation time when restoring the realtime clock. - Replace `sessionEndInputSaved` in the app-server protocol with `sessionInputSaved` and a `reason` of `chunk` or `sessionEnd`. Update the generated schemas and bindings. ## Testing Add coverage for reconstructing both receipt types, preserving rollout order in paginated realtime history, and restoring observation time across a restart after wall-clock rollback. GitOrigin-RevId: fcb4e0e539f4e4066726e83fa1e945fb659a2ca8
This commit is contained in:
@@ -6341,6 +6341,9 @@
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@@ -6349,19 +6352,20 @@
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
"sessionInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"title": "SessionInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"reason",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"title": "SessionInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
@@ -6485,6 +6489,13 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeSessionInputReason": {
|
||||
"enum": [
|
||||
"chunk",
|
||||
"sessionEnd"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeSessionOutcome": {
|
||||
"enum": [
|
||||
"ended",
|
||||
|
||||
+14
-3
@@ -25871,6 +25871,9 @@
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/v2/ThreadRealtimeSessionInputReason"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@@ -25879,19 +25882,20 @@
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
"sessionInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"title": "SessionInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"reason",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"title": "SessionInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
@@ -26045,6 +26049,13 @@
|
||||
"title": "ThreadRealtimeSdpNotification",
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeSessionInputReason": {
|
||||
"enum": [
|
||||
"chunk",
|
||||
"sessionEnd"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeSessionOutcome": {
|
||||
"enum": [
|
||||
"ended",
|
||||
|
||||
+14
-3
@@ -23567,6 +23567,9 @@
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@@ -23575,19 +23578,20 @@
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
"sessionInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"title": "SessionInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"reason",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"title": "SessionInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
@@ -23741,6 +23745,13 @@
|
||||
"title": "ThreadRealtimeSdpNotification",
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeSessionInputReason": {
|
||||
"enum": [
|
||||
"chunk",
|
||||
"sessionEnd"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeSessionOutcome": {
|
||||
"enum": [
|
||||
"ended",
|
||||
|
||||
+14
-3
@@ -241,6 +241,9 @@
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@@ -249,19 +252,20 @@
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
"sessionInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"title": "SessionInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"reason",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"title": "SessionInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
@@ -287,6 +291,13 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeSessionInputReason": {
|
||||
"enum": [
|
||||
"chunk",
|
||||
"sessionEnd"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeSessionOutcome": {
|
||||
"enum": [
|
||||
"ended",
|
||||
|
||||
+14
-3
@@ -241,6 +241,9 @@
|
||||
"input_item_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/ThreadRealtimeSessionInputReason"
|
||||
},
|
||||
"source_event_ids": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
@@ -249,19 +252,20 @@
|
||||
},
|
||||
"type": {
|
||||
"enum": [
|
||||
"sessionEndInputSaved"
|
||||
"sessionInputSaved"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItemType",
|
||||
"title": "SessionInputSavedThreadRealtimeItemType",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"input",
|
||||
"input_item_id",
|
||||
"reason",
|
||||
"source_event_ids",
|
||||
"type"
|
||||
],
|
||||
"title": "SessionEndInputSavedThreadRealtimeItem",
|
||||
"title": "SessionInputSavedThreadRealtimeItem",
|
||||
"type": "object"
|
||||
}
|
||||
],
|
||||
@@ -287,6 +291,13 @@
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"ThreadRealtimeSessionInputReason": {
|
||||
"enum": [
|
||||
"chunk",
|
||||
"sessionEnd"
|
||||
],
|
||||
"type": "string"
|
||||
},
|
||||
"ThreadRealtimeSessionOutcome": {
|
||||
"enum": [
|
||||
"ended",
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -4,6 +4,7 @@
|
||||
import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation";
|
||||
import type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels";
|
||||
import type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode";
|
||||
import type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason";
|
||||
import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
|
||||
import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole";
|
||||
|
||||
@@ -14,4 +15,4 @@ export type ThreadRealtimeItem = { id: string, realtimeSessionId: string,
|
||||
/**
|
||||
* Canonical observation time in Unix milliseconds for eligible voice calls.
|
||||
*/
|
||||
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionEndInputSaved", sourceEventIds: Array<string>, inputItemId: string, input: string, });
|
||||
observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array<string>, inputItemId: string, input: string, } | { "type": "sessionInputSaved", reason: ThreadRealtimeSessionInputReason, sourceEventIds: Array<string>, inputItemId: string, input: string, });
|
||||
|
||||
+5
@@ -0,0 +1,5 @@
|
||||
// GENERATED CODE! DO NOT MODIFY BY HAND!
|
||||
|
||||
// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
|
||||
|
||||
export type ThreadRealtimeSessionInputReason = "chunk" | "sessionEnd";
|
||||
@@ -599,6 +599,7 @@ export type { ThreadRealtimeItemTranscriptDeltaNotification } from "./ThreadReal
|
||||
export type { ThreadRealtimeOrder } from "./ThreadRealtimeOrder";
|
||||
export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification";
|
||||
export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification";
|
||||
export type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason";
|
||||
export type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome";
|
||||
export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport";
|
||||
export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification";
|
||||
|
||||
@@ -13,6 +13,7 @@ use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem;
|
||||
use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent;
|
||||
use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome;
|
||||
use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole;
|
||||
use codex_protocol::realtime::SessionInputReason as CoreSessionInputReason;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
@@ -66,7 +67,8 @@ pub enum ThreadRealtimeItemContent {
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
},
|
||||
SessionEndInputSaved {
|
||||
SessionInputSaved {
|
||||
reason: ThreadRealtimeSessionInputReason,
|
||||
source_event_ids: Vec<String>,
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
@@ -81,6 +83,14 @@ pub enum ThreadRealtimeHandoffMode {
|
||||
Steer,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[ts(rename_all = "camelCase", export_to = "v2/")]
|
||||
pub enum ThreadRealtimeSessionInputReason {
|
||||
Chunk,
|
||||
SessionEnd,
|
||||
}
|
||||
|
||||
/// EXPERIMENTAL - how an existing agent item appears in a realtime conversation.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
|
||||
#[serde(
|
||||
@@ -225,11 +235,18 @@ impl ThreadRealtimeItem {
|
||||
input_item_id,
|
||||
input,
|
||||
},
|
||||
CoreRealtimeItemContent::SessionEndInputSaved {
|
||||
CoreRealtimeItemContent::SessionInputSaved {
|
||||
reason,
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
} => ThreadRealtimeItemContent::SessionEndInputSaved {
|
||||
} => ThreadRealtimeItemContent::SessionInputSaved {
|
||||
reason: match reason {
|
||||
CoreSessionInputReason::Chunk => ThreadRealtimeSessionInputReason::Chunk,
|
||||
CoreSessionInputReason::SessionEnd => {
|
||||
ThreadRealtimeSessionInputReason::SessionEnd
|
||||
}
|
||||
},
|
||||
source_event_ids,
|
||||
input_item_id,
|
||||
input,
|
||||
|
||||
@@ -485,7 +485,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
if matches!(
|
||||
&item.content,
|
||||
codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. }
|
||||
| codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. }
|
||||
| codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. }
|
||||
) => {}
|
||||
RealtimeEvent::HistoryItemStarted(item) => {
|
||||
outgoing
|
||||
|
||||
@@ -28,6 +28,12 @@ fn observation_time_survives_clock_rollback_and_owner_restart() {
|
||||
observed_at_ms: Some(110),
|
||||
content: RealtimeItemContent::RealtimeSessionStarted,
|
||||
}),
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"type": "response_item",
|
||||
"payload": {"type": "message", "id": "msg_input_1", "role": "user", "content": [{"type": "input_text", "text": "hello"}]},
|
||||
"metadata": {"realtime_handoff": {"event_id": "receipt", "realtime_session_id": "voice", "observed_at_ms": 115, "handoff_id": "handoff", "mode": "start", "source_event_ids": []}}
|
||||
}))
|
||||
.expect("read annotated handoff input"),
|
||||
RolloutItem::RealtimeItem(RealtimeItem {
|
||||
id: "legacy".into(),
|
||||
realtime_session_id: "legacy-voice".into(),
|
||||
@@ -40,7 +46,7 @@ fn observation_time_survives_clock_rollback_and_owner_restart() {
|
||||
clock.observe(/*wall_clock_ms*/ 80),
|
||||
clock.observe(/*wall_clock_ms*/ 120)
|
||||
],
|
||||
[110, 120]
|
||||
[115, 120]
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,10 @@ use codex_protocol::protocol::ThreadSource;
|
||||
use codex_protocol::protocol::TokenUsageRecord;
|
||||
use codex_protocol::protocol::TurnContextItem;
|
||||
use codex_protocol::protocol::WorldStateItem;
|
||||
use codex_protocol::realtime::RealtimeHandoffMode;
|
||||
use codex_protocol::realtime::RealtimeItem;
|
||||
use codex_protocol::realtime::RealtimeItemContent;
|
||||
use codex_protocol::realtime::SessionInputReason;
|
||||
use codex_protocol::security_risk::SecurityRiskScore;
|
||||
use codex_protocol::turn_input::InjectedResponseItem;
|
||||
use codex_protocol::turn_input::ItemRetention;
|
||||
@@ -81,6 +84,14 @@ pub enum ModelSearchState {
|
||||
///
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)]
|
||||
pub struct CodexHarnessMetadata {
|
||||
/// Realtime provenance for an accepted ordinary turn input. Never model-visible.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub realtime_handoff: Option<Box<RealtimeHandoffMetadata>>,
|
||||
|
||||
/// Provenance of a transcript chunk saved in ordinary model history.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub realtime_session_input: Option<Box<RealtimeSessionInputMetadata>>,
|
||||
|
||||
/// Thread-local append sequence. Compaction keeps the identities of retained inputs.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub history_sequence: Option<u64>,
|
||||
@@ -200,6 +211,26 @@ pub struct CodexHarnessMetadata {
|
||||
pub sender_user_messages: Option<Box<SenderUserMessages>>,
|
||||
}
|
||||
|
||||
/// Facts that connect an ordinary accepted input to its originating realtime frames.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
|
||||
pub struct RealtimeHandoffMetadata {
|
||||
pub event_id: String,
|
||||
pub realtime_session_id: String,
|
||||
pub observed_at_ms: i64,
|
||||
pub handoff_id: String,
|
||||
pub mode: RealtimeHandoffMode,
|
||||
pub source_event_ids: Vec<String>,
|
||||
}
|
||||
|
||||
/// Realtime provenance for a saved transcript chunk; the response item owns its text.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)]
|
||||
pub struct RealtimeSessionInputMetadata {
|
||||
pub event_id: String,
|
||||
pub realtime_session_id: String,
|
||||
pub observed_at_ms: i64,
|
||||
pub source_event_ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, JsonSchema)]
|
||||
pub struct ImageReplaySource {
|
||||
pub prepared_url_digest: Vec<u8>,
|
||||
@@ -247,6 +278,70 @@ impl CodexHarnessMetadata {
|
||||
}
|
||||
|
||||
impl ResponseItemEnvelope {
|
||||
/// Projects the realtime view of an accepted input without storing a second copy.
|
||||
pub fn realtime_handoff_item(&self) -> Option<RealtimeItem> {
|
||||
let metadata = self.metadata.as_ref()?.realtime_handoff.as_ref()?;
|
||||
let ResponseItem::Message {
|
||||
id: Some(id),
|
||||
role,
|
||||
content,
|
||||
..
|
||||
} = &self.item
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let [ContentItem::InputText { text }] = content.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
if role != "user" {
|
||||
return None;
|
||||
}
|
||||
Some(RealtimeItem {
|
||||
id: metadata.event_id.clone(),
|
||||
realtime_session_id: metadata.realtime_session_id.clone(),
|
||||
observed_at_ms: Some(metadata.observed_at_ms),
|
||||
content: RealtimeItemContent::TurnInputAccepted {
|
||||
handoff_id: metadata.handoff_id.clone(),
|
||||
turn_id: self.item.turn_id()?.to_string(),
|
||||
mode: metadata.mode,
|
||||
source_event_ids: metadata.source_event_ids.clone(),
|
||||
input_item_id: id.to_string(),
|
||||
input: text.clone(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Projects a chunk receipt from its canonical model input.
|
||||
pub fn realtime_session_input_item(&self) -> Option<RealtimeItem> {
|
||||
let metadata = self.metadata.as_ref()?.realtime_session_input.as_ref()?;
|
||||
let ResponseItem::Message {
|
||||
id: Some(id),
|
||||
role,
|
||||
content,
|
||||
..
|
||||
} = &self.item
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
let [ContentItem::InputText { text }] = content.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
if role != "user" {
|
||||
return None;
|
||||
}
|
||||
Some(RealtimeItem {
|
||||
id: metadata.event_id.clone(),
|
||||
realtime_session_id: metadata.realtime_session_id.clone(),
|
||||
observed_at_ms: Some(metadata.observed_at_ms),
|
||||
content: RealtimeItemContent::SessionInputSaved {
|
||||
reason: SessionInputReason::Chunk,
|
||||
source_event_ids: metadata.source_event_ids.clone(),
|
||||
input_item_id: id.to_string(),
|
||||
input: text.clone(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the client requested that this item survive context-window changes and forks.
|
||||
pub fn is_always_retained(&self) -> bool {
|
||||
self.metadata
|
||||
@@ -337,6 +432,20 @@ impl RolloutItem {
|
||||
pub fn realtime_observed_at_ms(&self) -> Option<i64> {
|
||||
match self {
|
||||
Self::RealtimeItem(item) => item.observed_at_ms,
|
||||
Self::ResponseItem(envelope) => envelope
|
||||
.metadata
|
||||
.as_ref()?
|
||||
.realtime_handoff
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.observed_at_ms)
|
||||
.or_else(|| {
|
||||
envelope
|
||||
.metadata
|
||||
.as_ref()?
|
||||
.realtime_session_input
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.observed_at_ms)
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -345,6 +454,10 @@ impl RolloutItem {
|
||||
pub fn realtime_item(&self) -> Option<Cow<'_, RealtimeItem>> {
|
||||
match self {
|
||||
Self::RealtimeItem(item) => Some(Cow::Borrowed(item)),
|
||||
Self::ResponseItem(envelope) => envelope
|
||||
.realtime_handoff_item()
|
||||
.or_else(|| envelope.realtime_session_input_item())
|
||||
.map(Cow::Owned),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,98 @@ use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn accepted_realtime_input_projects_from_the_canonical_response_item() -> Result<()> {
|
||||
let mut item: ResponseItem = serde_json::from_value(json!({
|
||||
"type": "message",
|
||||
"id": "msg_input_123",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "use the smaller logo"}]
|
||||
}))?;
|
||||
item.set_turn_id_if_missing("turn-1");
|
||||
let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item,
|
||||
metadata: Some(CodexHarnessMetadata {
|
||||
realtime_handoff: Some(Box::new(RealtimeHandoffMetadata {
|
||||
event_id: "receipt-1".to_string(),
|
||||
realtime_session_id: "call-1".to_string(),
|
||||
observed_at_ms: 42,
|
||||
handoff_id: "handoff-1".to_string(),
|
||||
mode: RealtimeHandoffMode::Start,
|
||||
source_event_ids: vec!["frame-1".to_string()],
|
||||
})),
|
||||
..Default::default()
|
||||
}),
|
||||
});
|
||||
let persisted = serde_json::to_value(&rollout)?;
|
||||
assert_eq!(persisted["type"], "response_item");
|
||||
assert_eq!(
|
||||
persisted["payload"]["content"][0]["text"],
|
||||
"use the smaller logo"
|
||||
);
|
||||
assert_eq!(
|
||||
persisted["metadata"]["realtime_handoff"]["event_id"],
|
||||
"receipt-1"
|
||||
);
|
||||
let restored: RolloutItem = serde_json::from_value(persisted)?;
|
||||
assert_eq!(
|
||||
restored.realtime_item().map(Cow::into_owned),
|
||||
Some(RealtimeItem {
|
||||
id: "receipt-1".to_string(),
|
||||
realtime_session_id: "call-1".to_string(),
|
||||
observed_at_ms: Some(42),
|
||||
content: RealtimeItemContent::TurnInputAccepted {
|
||||
handoff_id: "handoff-1".to_string(),
|
||||
turn_id: "turn-1".to_string(),
|
||||
mode: RealtimeHandoffMode::Start,
|
||||
source_event_ids: vec!["frame-1".to_string()],
|
||||
input_item_id: "msg_input_123".to_string(),
|
||||
input: "use the smaller logo".to_string(),
|
||||
},
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcript_chunk_projects_from_its_only_model_input() -> Result<()> {
|
||||
let mut item: ResponseItem = serde_json::from_value(json!({
|
||||
"type": "message", "id": "msg_chunk", "role": "user",
|
||||
"content": [{"type": "input_text", "text": "user: hello"}]
|
||||
}))?;
|
||||
item.set_turn_id_if_missing("turn-1");
|
||||
let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item,
|
||||
metadata: Some(CodexHarnessMetadata {
|
||||
realtime_session_input: Some(Box::new(RealtimeSessionInputMetadata {
|
||||
event_id: "receipt-1".to_string(),
|
||||
realtime_session_id: "call-1".to_string(),
|
||||
observed_at_ms: 42,
|
||||
source_event_ids: vec!["frame-1".to_string()],
|
||||
})),
|
||||
..Default::default()
|
||||
}),
|
||||
});
|
||||
let persisted = serde_json::to_value(&rollout)?;
|
||||
assert_eq!(persisted["type"], "response_item");
|
||||
assert_eq!(persisted["payload"]["content"][0]["text"], "user: hello");
|
||||
assert_eq!(
|
||||
rollout.realtime_item().map(Cow::into_owned),
|
||||
Some(RealtimeItem {
|
||||
id: "receipt-1".to_string(),
|
||||
realtime_session_id: "call-1".to_string(),
|
||||
observed_at_ms: Some(42),
|
||||
content: RealtimeItemContent::SessionInputSaved {
|
||||
reason: SessionInputReason::Chunk,
|
||||
source_event_ids: vec!["frame-1".to_string()],
|
||||
input_item_id: "msg_chunk".to_string(),
|
||||
input: "user: hello".to_string(),
|
||||
},
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn thread_settings_snapshot(disabled_plugin_ids: Vec<String>) -> Result<ThreadSettingsSnapshot> {
|
||||
Ok(serde_json::from_value(json!({
|
||||
"model": "gpt-5",
|
||||
|
||||
@@ -63,14 +63,23 @@ pub enum RealtimeItemContent {
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
},
|
||||
/// End-of-call input recorded in model history without starting or steering a turn.
|
||||
SessionEndInputSaved {
|
||||
/// Transcript input saved in model history without starting or steering an agent turn.
|
||||
SessionInputSaved {
|
||||
reason: SessionInputReason,
|
||||
source_event_ids: Vec<String>,
|
||||
input_item_id: String,
|
||||
input: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[ts(rename_all = "snake_case")]
|
||||
pub enum SessionInputReason {
|
||||
Chunk,
|
||||
SessionEnd,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[ts(rename_all = "snake_case")]
|
||||
|
||||
@@ -285,7 +285,7 @@ fn rollout_item_type(item: &RolloutItem) -> String {
|
||||
codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. } => {
|
||||
"realtime.turn_input_accepted".to_string()
|
||||
}
|
||||
codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. } => {
|
||||
codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. } => {
|
||||
"realtime.session_end_input_saved".to_string()
|
||||
}
|
||||
},
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::time::Duration;
|
||||
use chrono::Utc;
|
||||
use codex_app_server_protocol::McpToolCallResult;
|
||||
use codex_app_server_protocol::ThreadItemPayload;
|
||||
use codex_protocol::ResponseItemId;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::AgentMessageContent;
|
||||
use codex_protocol::items::AgentMessageItem;
|
||||
@@ -41,6 +42,7 @@ use codex_protocol::realtime::RealtimeItem;
|
||||
use codex_protocol::realtime::RealtimeItemContent;
|
||||
use codex_protocol::realtime::RealtimeSessionOutcome;
|
||||
use codex_protocol::realtime::RealtimeTranscriptRole;
|
||||
use codex_rollout::ResponseItemEnvelope;
|
||||
use codex_rollout::RolloutConfig;
|
||||
use codex_rollout::RolloutItem;
|
||||
use codex_rollout::RolloutLine;
|
||||
@@ -619,6 +621,35 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
|
||||
.await
|
||||
.expect("persist thread metadata");
|
||||
|
||||
let mut handoff_input = ResponseItem::Message {
|
||||
encrypted_content: None,
|
||||
status: None,
|
||||
id: Some(ResponseItemId::from_server("msg_input_handoff".to_string())),
|
||||
role: "user".to_string(),
|
||||
content: vec![ContentItem::InputText {
|
||||
text: "use the smaller logo".to_string(),
|
||||
}],
|
||||
phase: None,
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
};
|
||||
handoff_input.set_turn_id_if_missing("turn-1");
|
||||
let handoff_input = RolloutItem::ResponseItem(ResponseItemEnvelope {
|
||||
item: handoff_input,
|
||||
metadata: Some(
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"realtime_handoff": {
|
||||
"event_id": "voice:handoff",
|
||||
"realtime_session_id": "voice",
|
||||
"observed_at_ms": 42,
|
||||
"handoff_id": "handoff",
|
||||
"mode": "start",
|
||||
"source_event_ids": ["voice:transcript:0"]
|
||||
}
|
||||
}))
|
||||
.expect("deserialize handoff provenance"),
|
||||
),
|
||||
});
|
||||
|
||||
store
|
||||
.append_items(AppendThreadItemsParams {
|
||||
thread_id,
|
||||
@@ -639,6 +670,7 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
|
||||
},
|
||||
}),
|
||||
turn_started("turn-1"),
|
||||
handoff_input,
|
||||
RolloutItem::RealtimeItem(RealtimeItem {
|
||||
id: "voice:promoted:agent-1".to_string(),
|
||||
realtime_session_id: "voice".to_string(),
|
||||
@@ -685,22 +717,23 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
|
||||
.expect("read canonical rollout")
|
||||
.lines()
|
||||
.map(|line| codex_rollout::parse_rollout_line(line).expect("parse rollout line"))
|
||||
.filter_map(|line| match line.item {
|
||||
RolloutItem::RealtimeItem(item) => Some((
|
||||
item.id,
|
||||
i64::try_from(line.ordinal.expect("paginated rollout ordinal"))
|
||||
.expect("ordinal fits SQLite integer"),
|
||||
chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str())
|
||||
.expect("valid rollout timestamp")
|
||||
.timestamp_millis(),
|
||||
serde_json::to_value(item.content)
|
||||
.expect("serialize realtime content")
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.expect("realtime content discriminator")
|
||||
.to_string(),
|
||||
)),
|
||||
_ => None,
|
||||
.filter_map(|line| {
|
||||
line.item.realtime_item().map(|item| {
|
||||
(
|
||||
item.id.clone(),
|
||||
i64::try_from(line.ordinal.expect("paginated rollout ordinal"))
|
||||
.expect("ordinal fits SQLite integer"),
|
||||
chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str())
|
||||
.expect("valid rollout timestamp")
|
||||
.timestamp_millis(),
|
||||
serde_json::to_value(&item.content)
|
||||
.expect("serialize realtime content")
|
||||
.get("type")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.expect("realtime content discriminator")
|
||||
.to_string(),
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
@@ -711,8 +744,9 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() {
|
||||
vec![
|
||||
("voice:started", 1, "realtime_session_started"),
|
||||
("voice:transcript:0", 2, "transcript_segment"),
|
||||
("voice:promoted:agent-1", 4, "bem_item_promoted"),
|
||||
("voice:closed", 6, "realtime_session_closed"),
|
||||
("voice:handoff", 4, "turn_input_accepted"),
|
||||
("voice:promoted:agent-1", 5, "bem_item_promoted"),
|
||||
("voice:closed", 7, "realtime_session_closed"),
|
||||
]
|
||||
);
|
||||
|
||||
|
||||
@@ -6520,27 +6520,6 @@ class TurnInputAcceptedThreadRealtimeItem(BaseModel):
|
||||
]
|
||||
|
||||
|
||||
class SessionEndInputSavedThreadRealtimeItem(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
)
|
||||
id: str
|
||||
observed_at_ms: Annotated[
|
||||
int | None,
|
||||
Field(
|
||||
alias="observedAtMs",
|
||||
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
),
|
||||
] = None
|
||||
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
|
||||
input: str
|
||||
input_item_id: str
|
||||
source_event_ids: list[str]
|
||||
type: Annotated[
|
||||
Literal["sessionEndInputSaved"], Field(title="SessionEndInputSavedThreadRealtimeItemType")
|
||||
]
|
||||
|
||||
|
||||
class ThreadRealtimeItemAddedNotification(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
@@ -6579,6 +6558,11 @@ class ThreadRealtimeSdpNotification(BaseModel):
|
||||
thread_id: Annotated[str, Field(alias="threadId")]
|
||||
|
||||
|
||||
class ThreadRealtimeSessionInputReason(Enum):
|
||||
chunk = "chunk"
|
||||
session_end = "sessionEnd"
|
||||
|
||||
|
||||
class ThreadRealtimeSessionOutcome(Enum):
|
||||
ended = "ended"
|
||||
failed = "failed"
|
||||
@@ -11405,6 +11389,28 @@ class RealtimeSessionClosedThreadRealtimeItem(BaseModel):
|
||||
]
|
||||
|
||||
|
||||
class SessionInputSavedThreadRealtimeItem(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
populate_by_name=True,
|
||||
)
|
||||
id: str
|
||||
observed_at_ms: Annotated[
|
||||
int | None,
|
||||
Field(
|
||||
alias="observedAtMs",
|
||||
description="Canonical observation time in Unix milliseconds for eligible voice calls.",
|
||||
),
|
||||
] = None
|
||||
realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")]
|
||||
input: str
|
||||
input_item_id: str
|
||||
reason: ThreadRealtimeSessionInputReason
|
||||
source_event_ids: list[str]
|
||||
type: Annotated[
|
||||
Literal["sessionInputSaved"], Field(title="SessionInputSavedThreadRealtimeItemType")
|
||||
]
|
||||
|
||||
|
||||
class ThreadRealtimeItem(
|
||||
RootModel[
|
||||
RealtimeSessionStartedThreadRealtimeItem
|
||||
@@ -11413,7 +11419,7 @@ class ThreadRealtimeItem(
|
||||
| BemItemPromotedThreadRealtimeItem
|
||||
| RealtimeSessionClosedThreadRealtimeItem
|
||||
| TurnInputAcceptedThreadRealtimeItem
|
||||
| SessionEndInputSavedThreadRealtimeItem
|
||||
| SessionInputSavedThreadRealtimeItem
|
||||
]
|
||||
):
|
||||
root: Annotated[
|
||||
@@ -11423,7 +11429,7 @@ class ThreadRealtimeItem(
|
||||
| BemItemPromotedThreadRealtimeItem
|
||||
| RealtimeSessionClosedThreadRealtimeItem
|
||||
| TurnInputAcceptedThreadRealtimeItem
|
||||
| SessionEndInputSavedThreadRealtimeItem,
|
||||
| SessionInputSavedThreadRealtimeItem,
|
||||
Field(
|
||||
description="EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline."
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user