From bd4147c2a42b8eee8a8fad75da26333b7816288d Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Mon, 20 Apr 2026 17:03:31 -0700 Subject: [PATCH] analytics: add responses api call schema --- .../analytics/src/analytics_client_tests.rs | 5 + codex-rs/analytics/src/client.rs | 1 + codex-rs/analytics/src/events.rs | 41 +++ codex-rs/analytics/src/facts.rs | 71 ++++ codex-rs/analytics/src/lib.rs | 29 ++ codex-rs/analytics/src/reducer.rs | 1 + codex-rs/analytics/src/response_items.rs | 346 ++++++++++++++++++ 7 files changed, 494 insertions(+) create mode 100644 codex-rs/analytics/src/response_items.rs diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index bf97da4284..7589c0bd37 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -29,6 +29,11 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexResponseItemType; +use crate::facts::CodexResponsesApiCallFact; +use crate::facts::CodexResponsesApiCallStatus; +use crate::facts::CodexResponsesApiItemMetadata; +use crate::facts::CodexResponsesApiItemPhase; use crate::facts::CompactionImplementation; use crate::facts::CompactionPhase; use crate::facts::CompactionReason; diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 25ab40953f..43aa200000 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -8,6 +8,7 @@ use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; +use crate::facts::CodexResponsesApiCallFact; use crate::facts::CustomAnalyticsFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index b542f8268f..8a1b7d2046 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1,5 +1,7 @@ use crate::facts::AppInvocation; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexResponsesApiCallStatus; +use crate::facts::CodexResponsesApiItemMetadata; use crate::facts::CompactionImplementation; use crate::facts::CompactionPhase; use crate::facts::CompactionReason; @@ -55,6 +57,8 @@ pub(crate) enum TrackEventRequest { AppUsed(CodexAppUsedEventRequest), HookRun(CodexHookRunEventRequest), Compaction(Box), + #[allow(dead_code)] + ResponsesApiCall(Box), TurnEvent(Box), TurnSteer(CodexTurnSteerEventRequest), PluginUsed(CodexPluginUsedEventRequest), @@ -311,6 +315,43 @@ pub(crate) struct CodexCompactionEventRequest { pub(crate) event_params: CodexCompactionEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexResponsesApiCallEventParams { + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) ephemeral: bool, + pub(crate) thread_source: Option, + pub(crate) initialization_mode: ThreadInitializationMode, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) responses_id: Option, + pub(crate) turn_responses_call_index: u64, + pub(crate) model: Option, + pub(crate) model_provider: Option, + pub(crate) reasoning_effort: Option, + pub(crate) status: CodexResponsesApiCallStatus, + pub(crate) error: Option, + pub(crate) started_at: u64, + pub(crate) completed_at: Option, + pub(crate) duration_ms: Option, + pub(crate) input_item_count: usize, + pub(crate) output_item_count: usize, + pub(crate) input_tokens: Option, + pub(crate) cached_input_tokens: Option, + pub(crate) output_tokens: Option, + pub(crate) reasoning_output_tokens: Option, + pub(crate) total_tokens: Option, + pub(crate) items: Vec, +} + +#[derive(Serialize)] +pub(crate) struct CodexResponsesApiCallEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexResponsesApiCallEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexTurnEventParams { pub(crate) thread_id: String, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 1d371acb1c..1609224fb6 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -13,6 +13,7 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ServiceTier; +use codex_protocol::models::MessagePhase; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::HookEventName; @@ -262,6 +263,76 @@ pub struct CodexCompactionEvent { pub duration_ms: Option, } +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CodexResponsesApiCallStatus { + Completed, + Failed, + Interrupted, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum CodexResponsesApiItemPhase { + Input, + Output, +} + +#[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CodexResponseItemType { + Message, + Reasoning, + LocalShellCall, + FunctionCall, + FunctionCallOutput, + CustomToolCall, + CustomToolCallOutput, + ToolSearchCall, + ToolSearchOutput, + WebSearchCall, + ImageGenerationCall, + GhostSnapshot, + Compaction, + Other, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +pub struct CodexResponsesApiItemMetadata { + pub item_phase: CodexResponsesApiItemPhase, + pub item_index: usize, + pub response_item_type: CodexResponseItemType, + pub role: Option, + pub status: Option, + pub message_phase: Option, + pub call_id: Option, + pub tool_name: Option, + pub payload_bytes: Option, + pub text_part_count: Option, + pub image_part_count: Option, +} + +#[derive(Clone, Debug)] +pub struct CodexResponsesApiCallFact { + pub thread_id: String, + pub turn_id: String, + pub responses_id: Option, + pub turn_responses_call_index: u64, + pub status: CodexResponsesApiCallStatus, + pub error: Option, + pub started_at: u64, + pub completed_at: Option, + pub duration_ms: Option, + pub input_item_count: usize, + pub output_item_count: usize, + pub input_tokens: Option, + pub cached_input_tokens: Option, + pub output_tokens: Option, + pub reasoning_output_tokens: Option, + pub total_tokens: Option, + pub items: Vec, +} + #[allow(dead_code)] pub(crate) enum AnalyticsFact { Initialize { diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 5c4cdfac79..13d1c056e4 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -2,7 +2,9 @@ mod client; mod events; mod facts; mod reducer; +mod response_items; +use serde::Serialize; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -18,6 +20,11 @@ pub use events::GuardianReviewedAction; pub use facts::AnalyticsJsonRpcError; pub use facts::AppInvocation; pub use facts::CodexCompactionEvent; +pub use facts::CodexResponseItemType; +pub use facts::CodexResponsesApiCallFact; +pub use facts::CodexResponsesApiCallStatus; +pub use facts::CodexResponsesApiItemMetadata; +pub use facts::CodexResponsesApiItemPhase; pub use facts::CodexTurnSteerEvent; pub use facts::CompactionImplementation; pub use facts::CompactionPhase; @@ -39,6 +46,7 @@ pub use facts::TurnSteerRequestError; pub use facts::TurnSteerResult; pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; +pub use response_items::response_items_metadata; #[cfg(test)] mod analytics_client_tests; @@ -49,3 +57,24 @@ pub fn now_unix_seconds() -> u64 { .unwrap_or_default() .as_secs() } + +pub(crate) fn serialized_string(value: &T) -> Option { + match serde_json::to_value(value).ok()? { + serde_json::Value::String(value) => Some(value), + value => Some(value.to_string()), + } +} + +pub(crate) fn serialized_bytes(value: &T) -> Option { + serde_json::to_string(value) + .ok() + .map(|value| byte_len(&value)) +} + +pub(crate) fn nonzero_i64(value: i64) -> Option { + (value > 0).then_some(value) +} + +pub(crate) fn byte_len(value: &str) -> i64 { + i64::try_from(value.len()).unwrap_or(i64::MAX) +} diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index a6ce3fc831..965b83c953 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -33,6 +33,7 @@ use crate::facts::AnalyticsJsonRpcError; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; +use crate::facts::CodexResponsesApiCallFact; use crate::facts::CustomAnalyticsFact; use crate::facts::HookRunInput; use crate::facts::PluginState; diff --git a/codex-rs/analytics/src/response_items.rs b/codex-rs/analytics/src/response_items.rs new file mode 100644 index 0000000000..e32be92900 --- /dev/null +++ b/codex-rs/analytics/src/response_items.rs @@ -0,0 +1,346 @@ +use crate::byte_len; +use crate::facts::CodexResponseItemType; +use crate::facts::CodexResponsesApiItemMetadata; +use crate::facts::CodexResponsesApiItemPhase; +use crate::nonzero_i64; +use crate::serialized_bytes; +use crate::serialized_string; +use codex_protocol::models::ContentItem; +use codex_protocol::models::FunctionCallOutputBody; +use codex_protocol::models::FunctionCallOutputContentItem; +use codex_protocol::models::FunctionCallOutputPayload; +use codex_protocol::models::ReasoningItemContent; +use codex_protocol::models::ReasoningItemReasoningSummary; +use codex_protocol::models::ResponseItem; + +pub fn response_items_metadata( + phase: CodexResponsesApiItemPhase, + items: &[ResponseItem], +) -> Vec { + items + .iter() + .enumerate() + .map(|(item_index, item)| response_item_metadata(phase, item_index, item)) + .collect() +} + +fn response_item_metadata( + item_phase: CodexResponsesApiItemPhase, + item_index: usize, + item: &ResponseItem, +) -> CodexResponsesApiItemMetadata { + let mut metadata = new_metadata(item_phase, item_index, response_item_type(item)); + + match item { + ResponseItem::Message { + role, + content, + phase, + .. + } => { + metadata.role = Some(role.clone()); + metadata.message_phase = phase.clone(); + metadata.payload_bytes = nonzero_i64(message_content_text_bytes(content)); + let (text_part_count, image_part_count) = message_content_part_counts(content); + metadata.text_part_count = Some(text_part_count); + metadata.image_part_count = Some(image_part_count); + } + ResponseItem::Reasoning { + summary, + content, + encrypted_content, + .. + } => { + metadata.payload_bytes = encrypted_content + .as_ref() + .map(|value| byte_len(value)) + .or_else(|| nonzero_i64(reasoning_content_bytes(summary, content))); + metadata.text_part_count = + Some(summary.len() + content.as_ref().map(std::vec::Vec::len).unwrap_or_default()); + metadata.image_part_count = Some(0); + } + ResponseItem::LocalShellCall { + call_id, + status, + action, + .. + } => { + metadata.call_id = call_id.clone(); + metadata.tool_name = Some("local_shell".to_string()); + metadata.status = serialized_string(status); + metadata.payload_bytes = serialized_bytes(action); + } + ResponseItem::FunctionCall { + name, + arguments, + call_id, + .. + } => { + metadata.call_id = Some(call_id.clone()); + metadata.tool_name = Some(name.clone()); + metadata.payload_bytes = Some(byte_len(arguments)); + } + ResponseItem::ToolSearchCall { + call_id, + status, + arguments, + .. + } => { + metadata.call_id = call_id.clone(); + metadata.tool_name = Some("tool_search".to_string()); + metadata.status = status.clone(); + metadata.payload_bytes = serialized_bytes(arguments); + } + ResponseItem::FunctionCallOutput { call_id, output } => { + metadata.call_id = Some(call_id.clone()); + metadata.payload_bytes = function_call_output_bytes(output); + let (text_part_count, image_part_count) = function_call_output_part_counts(output); + metadata.text_part_count = text_part_count; + metadata.image_part_count = image_part_count; + } + ResponseItem::CustomToolCall { + status, + call_id, + name, + input, + .. + } => { + metadata.call_id = Some(call_id.clone()); + metadata.tool_name = Some(name.clone()); + metadata.status = status.clone(); + metadata.payload_bytes = Some(byte_len(input)); + } + ResponseItem::CustomToolCallOutput { + call_id, + name, + output, + } => { + metadata.call_id = Some(call_id.clone()); + metadata.tool_name = name.clone(); + metadata.payload_bytes = function_call_output_bytes(output); + let (text_part_count, image_part_count) = function_call_output_part_counts(output); + metadata.text_part_count = text_part_count; + metadata.image_part_count = image_part_count; + } + ResponseItem::ToolSearchOutput { + call_id, + status, + tools, + .. + } => { + metadata.call_id = call_id.clone(); + metadata.tool_name = Some("tool_search".to_string()); + metadata.status = Some(status.clone()); + metadata.payload_bytes = serialized_bytes(tools); + } + ResponseItem::WebSearchCall { status, action, .. } => { + metadata.tool_name = Some("web_search".to_string()); + metadata.status = status.clone(); + metadata.payload_bytes = action.as_ref().and_then(serialized_bytes); + } + ResponseItem::ImageGenerationCall { + id, + status, + revised_prompt, + result, + } => { + metadata.call_id = Some(id.clone()); + metadata.tool_name = Some("image_generation".to_string()); + metadata.status = Some(status.clone()); + metadata.payload_bytes = nonzero_i64(byte_len(result)) + .or_else(|| revised_prompt.as_ref().map(|value| byte_len(value))); + } + ResponseItem::Compaction { encrypted_content } => { + metadata.payload_bytes = Some(byte_len(encrypted_content)); + } + ResponseItem::GhostSnapshot { .. } | ResponseItem::Other => {} + } + + metadata +} + +fn new_metadata( + item_phase: CodexResponsesApiItemPhase, + item_index: usize, + response_item_type: CodexResponseItemType, +) -> CodexResponsesApiItemMetadata { + CodexResponsesApiItemMetadata { + item_phase, + item_index, + response_item_type, + role: None, + status: None, + message_phase: None, + call_id: None, + tool_name: None, + payload_bytes: None, + text_part_count: None, + image_part_count: None, + } +} + +fn response_item_type(item: &ResponseItem) -> CodexResponseItemType { + match item { + ResponseItem::Message { .. } => CodexResponseItemType::Message, + ResponseItem::Reasoning { .. } => CodexResponseItemType::Reasoning, + ResponseItem::LocalShellCall { .. } => CodexResponseItemType::LocalShellCall, + ResponseItem::FunctionCall { .. } => CodexResponseItemType::FunctionCall, + ResponseItem::ToolSearchCall { .. } => CodexResponseItemType::ToolSearchCall, + ResponseItem::FunctionCallOutput { .. } => CodexResponseItemType::FunctionCallOutput, + ResponseItem::CustomToolCall { .. } => CodexResponseItemType::CustomToolCall, + ResponseItem::CustomToolCallOutput { .. } => CodexResponseItemType::CustomToolCallOutput, + ResponseItem::ToolSearchOutput { .. } => CodexResponseItemType::ToolSearchOutput, + ResponseItem::WebSearchCall { .. } => CodexResponseItemType::WebSearchCall, + ResponseItem::ImageGenerationCall { .. } => CodexResponseItemType::ImageGenerationCall, + ResponseItem::GhostSnapshot { .. } => CodexResponseItemType::GhostSnapshot, + ResponseItem::Compaction { .. } => CodexResponseItemType::Compaction, + ResponseItem::Other => CodexResponseItemType::Other, + } +} + +fn message_content_text_bytes(content: &[ContentItem]) -> i64 { + content + .iter() + .map(|item| match item { + ContentItem::InputText { text } | ContentItem::OutputText { text } => byte_len(text), + ContentItem::InputImage { .. } => 0, + }) + .sum() +} + +fn message_content_part_counts(content: &[ContentItem]) -> (usize, usize) { + let mut text_part_count = 0; + let mut image_part_count = 0; + for item in content { + match item { + ContentItem::InputText { .. } | ContentItem::OutputText { .. } => { + text_part_count += 1; + } + ContentItem::InputImage { .. } => { + image_part_count += 1; + } + } + } + (text_part_count, image_part_count) +} + +fn reasoning_content_bytes( + summary: &[ReasoningItemReasoningSummary], + content: &Option>, +) -> i64 { + let summary_bytes = summary + .iter() + .map(|summary| match summary { + ReasoningItemReasoningSummary::SummaryText { text } => byte_len(text), + }) + .sum::(); + let content_bytes = content + .as_ref() + .map(|content| { + content + .iter() + .map(|content| match content { + ReasoningItemContent::ReasoningText { text } + | ReasoningItemContent::Text { text } => byte_len(text), + }) + .sum::() + }) + .unwrap_or_default(); + summary_bytes + content_bytes +} + +fn function_call_output_bytes(output: &FunctionCallOutputPayload) -> Option { + match &output.body { + FunctionCallOutputBody::Text(text) => Some(byte_len(text)), + FunctionCallOutputBody::ContentItems(items) => serialized_bytes(items), + } +} + +fn function_call_output_part_counts( + output: &FunctionCallOutputPayload, +) -> (Option, Option) { + let Some(content_items) = output.content_items() else { + return (None, None); + }; + let mut text_part_count = 0; + let mut image_part_count = 0; + for item in content_items { + match item { + FunctionCallOutputContentItem::InputText { .. } => { + text_part_count += 1; + } + FunctionCallOutputContentItem::InputImage { .. } => { + image_part_count += 1; + } + } + } + (Some(text_part_count), Some(image_part_count)) +} + +#[cfg(test)] +mod tests { + use super::*; + use codex_protocol::models::ImageDetail; + use codex_protocol::models::MessagePhase; + + #[test] + fn maps_message_metadata() { + let items = vec![ResponseItem::Message { + id: None, + role: "assistant".to_string(), + content: vec![ + ContentItem::OutputText { + text: "hello".to_string(), + }, + ContentItem::InputImage { + image_url: "data:image/png;base64,abc".to_string(), + detail: None, + }, + ], + end_turn: None, + phase: Some(MessagePhase::FinalAnswer), + }]; + + let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items); + + assert_eq!(metadata[0].item_phase, CodexResponsesApiItemPhase::Output); + assert_eq!( + metadata[0].response_item_type, + CodexResponseItemType::Message + ); + assert_eq!(metadata[0].role.as_deref(), Some("assistant")); + assert_eq!(metadata[0].message_phase, Some(MessagePhase::FinalAnswer)); + assert_eq!(metadata[0].payload_bytes, Some(5)); + assert_eq!(metadata[0].text_part_count, Some(1)); + assert_eq!(metadata[0].image_part_count, Some(1)); + } + + #[test] + fn maps_tool_call_output_metadata() { + let items = vec![ResponseItem::CustomToolCallOutput { + call_id: "call_1".to_string(), + name: Some("custom_tool".to_string()), + output: FunctionCallOutputPayload::from_content_items(vec![ + FunctionCallOutputContentItem::InputText { + text: "result".to_string(), + }, + FunctionCallOutputContentItem::InputImage { + image_url: "https://example.test/image.png".to_string(), + detail: Some(ImageDetail::High), + }, + ]), + }]; + + let metadata = response_items_metadata(CodexResponsesApiItemPhase::Output, &items); + + assert_eq!( + metadata[0].response_item_type, + CodexResponseItemType::CustomToolCallOutput + ); + assert_eq!(metadata[0].call_id.as_deref(), Some("call_1")); + assert_eq!(metadata[0].tool_name.as_deref(), Some("custom_tool")); + assert!(metadata[0].payload_bytes.unwrap_or_default() > 0); + assert_eq!(metadata[0].text_part_count, Some(1)); + assert_eq!(metadata[0].image_part_count, Some(1)); + } +}