diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index fbfb065b7a..70e34a5c18 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -6341,6 +6341,9 @@ "input_item_id": { "type": "string" }, + "reason": { + "$ref": "#/definitions/ThreadRealtimeSessionInputReason" + }, "source_event_ids": { "items": { "type": "string" @@ -6349,19 +6352,20 @@ }, "type": { "enum": [ - "sessionEndInputSaved" + "sessionInputSaved" ], - "title": "SessionEndInputSavedThreadRealtimeItemType", + "title": "SessionInputSavedThreadRealtimeItemType", "type": "string" } }, "required": [ "input", "input_item_id", + "reason", "source_event_ids", "type" ], - "title": "SessionEndInputSavedThreadRealtimeItem", + "title": "SessionInputSavedThreadRealtimeItem", "type": "object" } ], @@ -6485,6 +6489,13 @@ ], "type": "object" }, + "ThreadRealtimeSessionInputReason": { + "enum": [ + "chunk", + "sessionEnd" + ], + "type": "string" + }, "ThreadRealtimeSessionOutcome": { "enum": [ "ended", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 3e4ca20411..3d138c25ae 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -25871,6 +25871,9 @@ "input_item_id": { "type": "string" }, + "reason": { + "$ref": "#/definitions/v2/ThreadRealtimeSessionInputReason" + }, "source_event_ids": { "items": { "type": "string" @@ -25879,19 +25882,20 @@ }, "type": { "enum": [ - "sessionEndInputSaved" + "sessionInputSaved" ], - "title": "SessionEndInputSavedThreadRealtimeItemType", + "title": "SessionInputSavedThreadRealtimeItemType", "type": "string" } }, "required": [ "input", "input_item_id", + "reason", "source_event_ids", "type" ], - "title": "SessionEndInputSavedThreadRealtimeItem", + "title": "SessionInputSavedThreadRealtimeItem", "type": "object" } ], @@ -26045,6 +26049,13 @@ "title": "ThreadRealtimeSdpNotification", "type": "object" }, + "ThreadRealtimeSessionInputReason": { + "enum": [ + "chunk", + "sessionEnd" + ], + "type": "string" + }, "ThreadRealtimeSessionOutcome": { "enum": [ "ended", diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index 5bb1062b19..eed0e8ef7f 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -23567,6 +23567,9 @@ "input_item_id": { "type": "string" }, + "reason": { + "$ref": "#/definitions/ThreadRealtimeSessionInputReason" + }, "source_event_ids": { "items": { "type": "string" @@ -23575,19 +23578,20 @@ }, "type": { "enum": [ - "sessionEndInputSaved" + "sessionInputSaved" ], - "title": "SessionEndInputSavedThreadRealtimeItemType", + "title": "SessionInputSavedThreadRealtimeItemType", "type": "string" } }, "required": [ "input", "input_item_id", + "reason", "source_event_ids", "type" ], - "title": "SessionEndInputSavedThreadRealtimeItem", + "title": "SessionInputSavedThreadRealtimeItem", "type": "object" } ], @@ -23741,6 +23745,13 @@ "title": "ThreadRealtimeSdpNotification", "type": "object" }, + "ThreadRealtimeSessionInputReason": { + "enum": [ + "chunk", + "sessionEnd" + ], + "type": "string" + }, "ThreadRealtimeSessionOutcome": { "enum": [ "ended", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json index 6ff298dfc0..8edd31598e 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemCompletedNotification.json @@ -241,6 +241,9 @@ "input_item_id": { "type": "string" }, + "reason": { + "$ref": "#/definitions/ThreadRealtimeSessionInputReason" + }, "source_event_ids": { "items": { "type": "string" @@ -249,19 +252,20 @@ }, "type": { "enum": [ - "sessionEndInputSaved" + "sessionInputSaved" ], - "title": "SessionEndInputSavedThreadRealtimeItemType", + "title": "SessionInputSavedThreadRealtimeItemType", "type": "string" } }, "required": [ "input", "input_item_id", + "reason", "source_event_ids", "type" ], - "title": "SessionEndInputSavedThreadRealtimeItem", + "title": "SessionInputSavedThreadRealtimeItem", "type": "object" } ], @@ -287,6 +291,13 @@ ], "type": "object" }, + "ThreadRealtimeSessionInputReason": { + "enum": [ + "chunk", + "sessionEnd" + ], + "type": "string" + }, "ThreadRealtimeSessionOutcome": { "enum": [ "ended", diff --git a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json index 7e026514a2..59839f61b3 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/ThreadRealtimeItemStartedNotification.json @@ -241,6 +241,9 @@ "input_item_id": { "type": "string" }, + "reason": { + "$ref": "#/definitions/ThreadRealtimeSessionInputReason" + }, "source_event_ids": { "items": { "type": "string" @@ -249,19 +252,20 @@ }, "type": { "enum": [ - "sessionEndInputSaved" + "sessionInputSaved" ], - "title": "SessionEndInputSavedThreadRealtimeItemType", + "title": "SessionInputSavedThreadRealtimeItemType", "type": "string" } }, "required": [ "input", "input_item_id", + "reason", "source_event_ids", "type" ], - "title": "SessionEndInputSavedThreadRealtimeItem", + "title": "SessionInputSavedThreadRealtimeItem", "type": "object" } ], @@ -287,6 +291,13 @@ ], "type": "object" }, + "ThreadRealtimeSessionInputReason": { + "enum": [ + "chunk", + "sessionEnd" + ], + "type": "string" + }, "ThreadRealtimeSessionOutcome": { "enum": [ "ended", diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst index fc61f59219..12935c308c 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-experimental.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst index 9e8b63e874..95ea9c74ed 100644 Binary files a/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst and b/codex-rs/app-server-protocol/schema/precomputed/app-server-exports-stable.json.zst differ diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts index a3cd2707a2..31c43b74aa 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeItem.ts @@ -4,6 +4,7 @@ import type { ThreadRealtimeBemItemPresentation } from "./ThreadRealtimeBemItemPresentation"; import type { ThreadRealtimeFrameChannels } from "./ThreadRealtimeFrameChannels"; import type { ThreadRealtimeHandoffMode } from "./ThreadRealtimeHandoffMode"; +import type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason"; import type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome"; import type { ThreadRealtimeTranscriptRole } from "./ThreadRealtimeTranscriptRole"; @@ -14,4 +15,4 @@ export type ThreadRealtimeItem = { id: string, realtimeSessionId: string, /** * Canonical observation time in Unix milliseconds for eligible voice calls. */ -observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array, inputItemId: string, input: string, } | { "type": "sessionEndInputSaved", sourceEventIds: Array, inputItemId: string, input: string, }); +observedAtMs?: number, } & ({ "type": "realtimeSessionStarted" } | { "type": "frame", channels: ThreadRealtimeFrameChannels, } | { "type": "transcriptSegment", role: ThreadRealtimeTranscriptRole, text: string, } | { "type": "bemItemPromoted", turnId: string, itemId: string, presentation: ThreadRealtimeBemItemPresentation, } | { "type": "realtimeSessionClosed", outcome: ThreadRealtimeSessionOutcome, } | { "type": "turnInputAccepted", handoffId: string, turnId: string, mode: ThreadRealtimeHandoffMode, sourceEventIds: Array, inputItemId: string, input: string, } | { "type": "sessionInputSaved", reason: ThreadRealtimeSessionInputReason, sourceEventIds: Array, inputItemId: string, input: string, }); diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionInputReason.ts b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionInputReason.ts new file mode 100644 index 0000000000..95c91dfead --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/v2/ThreadRealtimeSessionInputReason.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type ThreadRealtimeSessionInputReason = "chunk" | "sessionEnd"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts index b0f9bc947b..a766a9c254 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/index.ts @@ -599,6 +599,7 @@ export type { ThreadRealtimeItemTranscriptDeltaNotification } from "./ThreadReal export type { ThreadRealtimeOrder } from "./ThreadRealtimeOrder"; export type { ThreadRealtimeOutputAudioDeltaNotification } from "./ThreadRealtimeOutputAudioDeltaNotification"; export type { ThreadRealtimeSdpNotification } from "./ThreadRealtimeSdpNotification"; +export type { ThreadRealtimeSessionInputReason } from "./ThreadRealtimeSessionInputReason"; export type { ThreadRealtimeSessionOutcome } from "./ThreadRealtimeSessionOutcome"; export type { ThreadRealtimeStartTransport } from "./ThreadRealtimeStartTransport"; export type { ThreadRealtimeStartedNotification } from "./ThreadRealtimeStartedNotification"; diff --git a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs index 987681d90d..5234e039c7 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2/realtime.rs @@ -13,6 +13,7 @@ use codex_protocol::realtime::RealtimeItem as CoreRealtimeItem; use codex_protocol::realtime::RealtimeItemContent as CoreRealtimeItemContent; use codex_protocol::realtime::RealtimeSessionOutcome as CoreRealtimeSessionOutcome; use codex_protocol::realtime::RealtimeTranscriptRole as CoreRealtimeTranscriptRole; +use codex_protocol::realtime::SessionInputReason as CoreSessionInputReason; use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; @@ -66,7 +67,8 @@ pub enum ThreadRealtimeItemContent { input_item_id: String, input: String, }, - SessionEndInputSaved { + SessionInputSaved { + reason: ThreadRealtimeSessionInputReason, source_event_ids: Vec, input_item_id: String, input: String, @@ -81,6 +83,14 @@ pub enum ThreadRealtimeHandoffMode { Steer, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "camelCase")] +#[ts(rename_all = "camelCase", export_to = "v2/")] +pub enum ThreadRealtimeSessionInputReason { + Chunk, + SessionEnd, +} + /// EXPERIMENTAL - how an existing agent item appears in a realtime conversation. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)] #[serde( @@ -225,11 +235,18 @@ impl ThreadRealtimeItem { input_item_id, input, }, - CoreRealtimeItemContent::SessionEndInputSaved { + CoreRealtimeItemContent::SessionInputSaved { + reason, source_event_ids, input_item_id, input, - } => ThreadRealtimeItemContent::SessionEndInputSaved { + } => ThreadRealtimeItemContent::SessionInputSaved { + reason: match reason { + CoreSessionInputReason::Chunk => ThreadRealtimeSessionInputReason::Chunk, + CoreSessionInputReason::SessionEnd => { + ThreadRealtimeSessionInputReason::SessionEnd + } + }, source_event_ids, input_item_id, input, diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index a321186b1d..c64be6aed7 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -485,7 +485,7 @@ pub(crate) async fn apply_bespoke_event_handling( if matches!( &item.content, codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. } - | codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. } + | codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. } ) => {} RealtimeEvent::HistoryItemStarted(item) => { outgoing diff --git a/codex-rs/core/src/observation_clock_tests.rs b/codex-rs/core/src/observation_clock_tests.rs index 60bc66e9f3..fcda411012 100644 --- a/codex-rs/core/src/observation_clock_tests.rs +++ b/codex-rs/core/src/observation_clock_tests.rs @@ -28,6 +28,12 @@ fn observation_time_survives_clock_rollback_and_owner_restart() { observed_at_ms: Some(110), content: RealtimeItemContent::RealtimeSessionStarted, }), + serde_json::from_value(serde_json::json!({ + "type": "response_item", + "payload": {"type": "message", "id": "msg_input_1", "role": "user", "content": [{"type": "input_text", "text": "hello"}]}, + "metadata": {"realtime_handoff": {"event_id": "receipt", "realtime_session_id": "voice", "observed_at_ms": 115, "handoff_id": "handoff", "mode": "start", "source_event_ids": []}} + })) + .expect("read annotated handoff input"), RolloutItem::RealtimeItem(RealtimeItem { id: "legacy".into(), realtime_session_id: "legacy-voice".into(), @@ -40,7 +46,7 @@ fn observation_time_survives_clock_rollback_and_owner_restart() { clock.observe(/*wall_clock_ms*/ 80), clock.observe(/*wall_clock_ms*/ 120) ], - [110, 120] + [115, 120] ); } diff --git a/codex-rs/history/src/lib.rs b/codex-rs/history/src/lib.rs index 7758f0b286..50ac2712f7 100644 --- a/codex-rs/history/src/lib.rs +++ b/codex-rs/history/src/lib.rs @@ -45,7 +45,10 @@ use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsageRecord; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::WorldStateItem; +use codex_protocol::realtime::RealtimeHandoffMode; use codex_protocol::realtime::RealtimeItem; +use codex_protocol::realtime::RealtimeItemContent; +use codex_protocol::realtime::SessionInputReason; use codex_protocol::security_risk::SecurityRiskScore; use codex_protocol::turn_input::InjectedResponseItem; use codex_protocol::turn_input::ItemRetention; @@ -81,6 +84,14 @@ pub enum ModelSearchState { /// #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, Eq, JsonSchema)] pub struct CodexHarnessMetadata { + /// Realtime provenance for an accepted ordinary turn input. Never model-visible. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub realtime_handoff: Option>, + + /// Provenance of a transcript chunk saved in ordinary model history. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub realtime_session_input: Option>, + /// Thread-local append sequence. Compaction keeps the identities of retained inputs. #[serde(default, skip_serializing_if = "Option::is_none")] pub history_sequence: Option, @@ -200,6 +211,26 @@ pub struct CodexHarnessMetadata { pub sender_user_messages: Option>, } +/// Facts that connect an ordinary accepted input to its originating realtime frames. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +pub struct RealtimeHandoffMetadata { + pub event_id: String, + pub realtime_session_id: String, + pub observed_at_ms: i64, + pub handoff_id: String, + pub mode: RealtimeHandoffMode, + pub source_event_ids: Vec, +} + +/// Realtime provenance for a saved transcript chunk; the response item owns its text. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema)] +pub struct RealtimeSessionInputMetadata { + pub event_id: String, + pub realtime_session_id: String, + pub observed_at_ms: i64, + pub source_event_ids: Vec, +} + #[derive(Serialize, Deserialize, Clone, PartialEq, Eq, JsonSchema)] pub struct ImageReplaySource { pub prepared_url_digest: Vec, @@ -247,6 +278,70 @@ impl CodexHarnessMetadata { } impl ResponseItemEnvelope { + /// Projects the realtime view of an accepted input without storing a second copy. + pub fn realtime_handoff_item(&self) -> Option { + let metadata = self.metadata.as_ref()?.realtime_handoff.as_ref()?; + let ResponseItem::Message { + id: Some(id), + role, + content, + .. + } = &self.item + else { + return None; + }; + let [ContentItem::InputText { text }] = content.as_slice() else { + return None; + }; + if role != "user" { + return None; + } + Some(RealtimeItem { + id: metadata.event_id.clone(), + realtime_session_id: metadata.realtime_session_id.clone(), + observed_at_ms: Some(metadata.observed_at_ms), + content: RealtimeItemContent::TurnInputAccepted { + handoff_id: metadata.handoff_id.clone(), + turn_id: self.item.turn_id()?.to_string(), + mode: metadata.mode, + source_event_ids: metadata.source_event_ids.clone(), + input_item_id: id.to_string(), + input: text.clone(), + }, + }) + } + + /// Projects a chunk receipt from its canonical model input. + pub fn realtime_session_input_item(&self) -> Option { + let metadata = self.metadata.as_ref()?.realtime_session_input.as_ref()?; + let ResponseItem::Message { + id: Some(id), + role, + content, + .. + } = &self.item + else { + return None; + }; + let [ContentItem::InputText { text }] = content.as_slice() else { + return None; + }; + if role != "user" { + return None; + } + Some(RealtimeItem { + id: metadata.event_id.clone(), + realtime_session_id: metadata.realtime_session_id.clone(), + observed_at_ms: Some(metadata.observed_at_ms), + content: RealtimeItemContent::SessionInputSaved { + reason: SessionInputReason::Chunk, + source_event_ids: metadata.source_event_ids.clone(), + input_item_id: id.to_string(), + input: text.clone(), + }, + }) + } + /// Whether the client requested that this item survive context-window changes and forks. pub fn is_always_retained(&self) -> bool { self.metadata @@ -337,6 +432,20 @@ impl RolloutItem { pub fn realtime_observed_at_ms(&self) -> Option { match self { Self::RealtimeItem(item) => item.observed_at_ms, + Self::ResponseItem(envelope) => envelope + .metadata + .as_ref()? + .realtime_handoff + .as_ref() + .map(|metadata| metadata.observed_at_ms) + .or_else(|| { + envelope + .metadata + .as_ref()? + .realtime_session_input + .as_ref() + .map(|metadata| metadata.observed_at_ms) + }), _ => None, } } @@ -345,6 +454,10 @@ impl RolloutItem { pub fn realtime_item(&self) -> Option> { match self { Self::RealtimeItem(item) => Some(Cow::Borrowed(item)), + Self::ResponseItem(envelope) => envelope + .realtime_handoff_item() + .or_else(|| envelope.realtime_session_input_item()) + .map(Cow::Owned), _ => None, } } diff --git a/codex-rs/history/src/tests.rs b/codex-rs/history/src/tests.rs index 267a732fb3..dee6147caf 100644 --- a/codex-rs/history/src/tests.rs +++ b/codex-rs/history/src/tests.rs @@ -16,6 +16,98 @@ use serde_json::json; use super::*; +#[test] +fn accepted_realtime_input_projects_from_the_canonical_response_item() -> Result<()> { + let mut item: ResponseItem = serde_json::from_value(json!({ + "type": "message", + "id": "msg_input_123", + "role": "user", + "content": [{"type": "input_text", "text": "use the smaller logo"}] + }))?; + item.set_turn_id_if_missing("turn-1"); + let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope { + item, + metadata: Some(CodexHarnessMetadata { + realtime_handoff: Some(Box::new(RealtimeHandoffMetadata { + event_id: "receipt-1".to_string(), + realtime_session_id: "call-1".to_string(), + observed_at_ms: 42, + handoff_id: "handoff-1".to_string(), + mode: RealtimeHandoffMode::Start, + source_event_ids: vec!["frame-1".to_string()], + })), + ..Default::default() + }), + }); + let persisted = serde_json::to_value(&rollout)?; + assert_eq!(persisted["type"], "response_item"); + assert_eq!( + persisted["payload"]["content"][0]["text"], + "use the smaller logo" + ); + assert_eq!( + persisted["metadata"]["realtime_handoff"]["event_id"], + "receipt-1" + ); + let restored: RolloutItem = serde_json::from_value(persisted)?; + assert_eq!( + restored.realtime_item().map(Cow::into_owned), + Some(RealtimeItem { + id: "receipt-1".to_string(), + realtime_session_id: "call-1".to_string(), + observed_at_ms: Some(42), + content: RealtimeItemContent::TurnInputAccepted { + handoff_id: "handoff-1".to_string(), + turn_id: "turn-1".to_string(), + mode: RealtimeHandoffMode::Start, + source_event_ids: vec!["frame-1".to_string()], + input_item_id: "msg_input_123".to_string(), + input: "use the smaller logo".to_string(), + }, + }) + ); + Ok(()) +} + +#[test] +fn transcript_chunk_projects_from_its_only_model_input() -> Result<()> { + let mut item: ResponseItem = serde_json::from_value(json!({ + "type": "message", "id": "msg_chunk", "role": "user", + "content": [{"type": "input_text", "text": "user: hello"}] + }))?; + item.set_turn_id_if_missing("turn-1"); + let rollout = RolloutItem::ResponseItem(ResponseItemEnvelope { + item, + metadata: Some(CodexHarnessMetadata { + realtime_session_input: Some(Box::new(RealtimeSessionInputMetadata { + event_id: "receipt-1".to_string(), + realtime_session_id: "call-1".to_string(), + observed_at_ms: 42, + source_event_ids: vec!["frame-1".to_string()], + })), + ..Default::default() + }), + }); + let persisted = serde_json::to_value(&rollout)?; + assert_eq!(persisted["type"], "response_item"); + assert_eq!(persisted["payload"]["content"][0]["text"], "user: hello"); + assert_eq!( + rollout.realtime_item().map(Cow::into_owned), + Some(RealtimeItem { + id: "receipt-1".to_string(), + realtime_session_id: "call-1".to_string(), + observed_at_ms: Some(42), + content: RealtimeItemContent::SessionInputSaved { + reason: SessionInputReason::Chunk, + source_event_ids: vec!["frame-1".to_string()], + input_item_id: "msg_chunk".to_string(), + input: "user: hello".to_string(), + }, + }) + ); + Ok(()) +} + fn thread_settings_snapshot(disabled_plugin_ids: Vec) -> Result { Ok(serde_json::from_value(json!({ "model": "gpt-5", diff --git a/codex-rs/protocol/src/realtime.rs b/codex-rs/protocol/src/realtime.rs index 8da13879c2..fde949c529 100644 --- a/codex-rs/protocol/src/realtime.rs +++ b/codex-rs/protocol/src/realtime.rs @@ -63,14 +63,23 @@ pub enum RealtimeItemContent { input_item_id: String, input: String, }, - /// End-of-call input recorded in model history without starting or steering a turn. - SessionEndInputSaved { + /// Transcript input saved in model history without starting or steering an agent turn. + SessionInputSaved { + reason: SessionInputReason, source_event_ids: Vec, input_item_id: String, input: String, }, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +#[ts(rename_all = "snake_case")] +pub enum SessionInputReason { + Chunk, + SessionEnd, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, JsonSchema, TS)] #[serde(rename_all = "snake_case")] #[ts(rename_all = "snake_case")] diff --git a/codex-rs/rollout/src/persistence_metrics.rs b/codex-rs/rollout/src/persistence_metrics.rs index 7cf5047bfe..f5ac624c17 100644 --- a/codex-rs/rollout/src/persistence_metrics.rs +++ b/codex-rs/rollout/src/persistence_metrics.rs @@ -285,7 +285,7 @@ fn rollout_item_type(item: &RolloutItem) -> String { codex_protocol::realtime::RealtimeItemContent::TurnInputAccepted { .. } => { "realtime.turn_input_accepted".to_string() } - codex_protocol::realtime::RealtimeItemContent::SessionEndInputSaved { .. } => { + codex_protocol::realtime::RealtimeItemContent::SessionInputSaved { .. } => { "realtime.session_end_input_saved".to_string() } }, diff --git a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs index 3cfa0e5030..c3dedf7411 100644 --- a/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs +++ b/codex-rs/thread-store/src/local/thread_history_materialization_tests.rs @@ -8,6 +8,7 @@ use std::time::Duration; use chrono::Utc; use codex_app_server_protocol::McpToolCallResult; use codex_app_server_protocol::ThreadItemPayload; +use codex_protocol::ResponseItemId; use codex_protocol::ThreadId; use codex_protocol::items::AgentMessageContent; use codex_protocol::items::AgentMessageItem; @@ -41,6 +42,7 @@ use codex_protocol::realtime::RealtimeItem; use codex_protocol::realtime::RealtimeItemContent; use codex_protocol::realtime::RealtimeSessionOutcome; use codex_protocol::realtime::RealtimeTranscriptRole; +use codex_rollout::ResponseItemEnvelope; use codex_rollout::RolloutConfig; use codex_rollout::RolloutItem; use codex_rollout::RolloutLine; @@ -619,6 +621,35 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() { .await .expect("persist thread metadata"); + let mut handoff_input = ResponseItem::Message { + encrypted_content: None, + status: None, + id: Some(ResponseItemId::from_server("msg_input_handoff".to_string())), + role: "user".to_string(), + content: vec![ContentItem::InputText { + text: "use the smaller logo".to_string(), + }], + phase: None, + internal_chat_message_metadata_passthrough: None, + }; + handoff_input.set_turn_id_if_missing("turn-1"); + let handoff_input = RolloutItem::ResponseItem(ResponseItemEnvelope { + item: handoff_input, + metadata: Some( + serde_json::from_value(serde_json::json!({ + "realtime_handoff": { + "event_id": "voice:handoff", + "realtime_session_id": "voice", + "observed_at_ms": 42, + "handoff_id": "handoff", + "mode": "start", + "source_event_ids": ["voice:transcript:0"] + } + })) + .expect("deserialize handoff provenance"), + ), + }); + store .append_items(AppendThreadItemsParams { thread_id, @@ -639,6 +670,7 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() { }, }), turn_started("turn-1"), + handoff_input, RolloutItem::RealtimeItem(RealtimeItem { id: "voice:promoted:agent-1".to_string(), realtime_session_id: "voice".to_string(), @@ -685,22 +717,23 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() { .expect("read canonical rollout") .lines() .map(|line| codex_rollout::parse_rollout_line(line).expect("parse rollout line")) - .filter_map(|line| match line.item { - RolloutItem::RealtimeItem(item) => Some(( - item.id, - i64::try_from(line.ordinal.expect("paginated rollout ordinal")) - .expect("ordinal fits SQLite integer"), - chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str()) - .expect("valid rollout timestamp") - .timestamp_millis(), - serde_json::to_value(item.content) - .expect("serialize realtime content") - .get("type") - .and_then(serde_json::Value::as_str) - .expect("realtime content discriminator") - .to_string(), - )), - _ => None, + .filter_map(|line| { + line.item.realtime_item().map(|item| { + ( + item.id.clone(), + i64::try_from(line.ordinal.expect("paginated rollout ordinal")) + .expect("ordinal fits SQLite integer"), + chrono::DateTime::parse_from_rfc3339(line.timestamp.as_str()) + .expect("valid rollout timestamp") + .timestamp_millis(), + serde_json::to_value(&item.content) + .expect("serialize realtime content") + .get("type") + .and_then(serde_json::Value::as_str) + .expect("realtime content discriminator") + .to_string(), + ) + }) }) .collect::>(); assert_eq!( @@ -711,8 +744,9 @@ async fn paginated_realtime_items_materialize_separately_in_rollout_order() { vec![ ("voice:started", 1, "realtime_session_started"), ("voice:transcript:0", 2, "transcript_segment"), - ("voice:promoted:agent-1", 4, "bem_item_promoted"), - ("voice:closed", 6, "realtime_session_closed"), + ("voice:handoff", 4, "turn_input_accepted"), + ("voice:promoted:agent-1", 5, "bem_item_promoted"), + ("voice:closed", 7, "realtime_session_closed"), ] ); diff --git a/sdk/python/src/openai_codex/generated/v2_all.py b/sdk/python/src/openai_codex/generated/v2_all.py index bfbcc486af..e6bd509b4c 100644 --- a/sdk/python/src/openai_codex/generated/v2_all.py +++ b/sdk/python/src/openai_codex/generated/v2_all.py @@ -6520,27 +6520,6 @@ class TurnInputAcceptedThreadRealtimeItem(BaseModel): ] -class SessionEndInputSavedThreadRealtimeItem(BaseModel): - model_config = ConfigDict( - populate_by_name=True, - ) - id: str - observed_at_ms: Annotated[ - int | None, - Field( - alias="observedAtMs", - description="Canonical observation time in Unix milliseconds for eligible voice calls.", - ), - ] = None - realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")] - input: str - input_item_id: str - source_event_ids: list[str] - type: Annotated[ - Literal["sessionEndInputSaved"], Field(title="SessionEndInputSavedThreadRealtimeItemType") - ] - - class ThreadRealtimeItemAddedNotification(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -6579,6 +6558,11 @@ class ThreadRealtimeSdpNotification(BaseModel): thread_id: Annotated[str, Field(alias="threadId")] +class ThreadRealtimeSessionInputReason(Enum): + chunk = "chunk" + session_end = "sessionEnd" + + class ThreadRealtimeSessionOutcome(Enum): ended = "ended" failed = "failed" @@ -11405,6 +11389,28 @@ class RealtimeSessionClosedThreadRealtimeItem(BaseModel): ] +class SessionInputSavedThreadRealtimeItem(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + id: str + observed_at_ms: Annotated[ + int | None, + Field( + alias="observedAtMs", + description="Canonical observation time in Unix milliseconds for eligible voice calls.", + ), + ] = None + realtime_session_id: Annotated[str, Field(alias="realtimeSessionId")] + input: str + input_item_id: str + reason: ThreadRealtimeSessionInputReason + source_event_ids: list[str] + type: Annotated[ + Literal["sessionInputSaved"], Field(title="SessionInputSavedThreadRealtimeItemType") + ] + + class ThreadRealtimeItem( RootModel[ RealtimeSessionStartedThreadRealtimeItem @@ -11413,7 +11419,7 @@ class ThreadRealtimeItem( | BemItemPromotedThreadRealtimeItem | RealtimeSessionClosedThreadRealtimeItem | TurnInputAcceptedThreadRealtimeItem - | SessionEndInputSavedThreadRealtimeItem + | SessionInputSavedThreadRealtimeItem ] ): root: Annotated[ @@ -11423,7 +11429,7 @@ class ThreadRealtimeItem( | BemItemPromotedThreadRealtimeItem | RealtimeSessionClosedThreadRealtimeItem | TurnInputAcceptedThreadRealtimeItem - | SessionEndInputSavedThreadRealtimeItem, + | SessionInputSavedThreadRealtimeItem, Field( description="EXPERIMENTAL - a thread-scoped realtime item in the canonical timeline." ),