From 355fb1c285e35fd443bc35af8dde6111e468243a Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 21 Apr 2026 09:17:56 -0700 Subject: [PATCH] analytics: reduce responses api call facts --- .../analytics/src/analytics_client_tests.rs | 200 ++++++++++++++++++ codex-rs/analytics/src/client.rs | 40 ++++ codex-rs/analytics/src/events.rs | 1 - codex-rs/analytics/src/facts.rs | 1 + codex-rs/analytics/src/reducer.rs | 90 ++++++++ 5 files changed, 331 insertions(+), 1 deletion(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 7589c0bd37..ab52079d7b 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -7,6 +7,8 @@ use crate::events::CodexCompactionEventRequest; use crate::events::CodexHookRunEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; +use crate::events::CodexResponsesApiCallEventParams; +use crate::events::CodexResponsesApiCallEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventRequest; use crate::events::GuardianApprovalRequestSource; @@ -96,6 +98,7 @@ use codex_plugin::PluginTelemetryMetadata; use codex_protocol::approvals::NetworkApprovalProtocol; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ModeKind; +use codex_protocol::models::MessagePhase; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::HookEventName; use codex_protocol::protocol::HookRunStatus; @@ -278,6 +281,25 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag } } +fn sample_responses_api_item_metadata( + item_phase: CodexResponsesApiItemPhase, + item_index: usize, +) -> CodexResponsesApiItemMetadata { + CodexResponsesApiItemMetadata { + item_phase, + item_index, + response_item_type: CodexResponseItemType::Message, + role: Some("assistant".to_string()), + status: Some("completed".to_string()), + message_phase: Some(MessagePhase::FinalAnswer), + call_id: None, + tool_name: None, + payload_bytes: Some(12), + text_part_count: Some(1), + image_part_count: Some(0), + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -757,6 +779,102 @@ fn compaction_event_serializes_expected_shape() { ); } +#[test] +fn responses_api_call_event_serializes_expected_shape() { + let event = TrackEventRequest::ResponsesApiCall(Box::new(CodexResponsesApiCallEventRequest { + event_type: "codex_responses_api_call_event", + event_params: CodexResponsesApiCallEventParams { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + ephemeral: false, + thread_source: Some("user".to_string()), + initialization_mode: ThreadInitializationMode::New, + subagent_source: None, + parent_thread_id: None, + app_server_client: sample_app_server_client_metadata(), + runtime: sample_runtime_metadata(), + responses_id: Some("resp_123".to_string()), + turn_responses_call_index: 2, + model: Some("gpt-5".to_string()), + model_provider: Some("openai".to_string()), + reasoning_effort: Some("high".to_string()), + status: CodexResponsesApiCallStatus::Completed, + error: None, + started_at: 100, + completed_at: Some(102), + duration_ms: Some(2345), + input_item_count: 1, + output_item_count: 1, + input_tokens: Some(10), + cached_input_tokens: Some(4), + output_tokens: Some(20), + reasoning_output_tokens: Some(3), + total_tokens: Some(30), + items: vec![ + sample_responses_api_item_metadata( + CodexResponsesApiItemPhase::Input, + /*item_index*/ 0, + ), + sample_responses_api_item_metadata( + CodexResponsesApiItemPhase::Output, + /*item_index*/ 0, + ), + ], + }, + })); + + let payload = serde_json::to_value(&event).expect("serialize responses api call event"); + + assert_eq!(payload["event_type"], "codex_responses_api_call_event"); + assert_eq!(payload["event_params"]["thread_id"], "thread-1"); + assert_eq!(payload["event_params"]["turn_id"], "turn-1"); + assert_eq!(payload["event_params"]["ephemeral"], false); + assert_eq!(payload["event_params"]["thread_source"], "user"); + assert_eq!(payload["event_params"]["initialization_mode"], "new"); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + DEFAULT_ORIGINATOR + ); + assert_eq!( + payload["event_params"]["app_server_client"]["rpc_transport"], + "stdio" + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + "0.1.0" + ); + assert_eq!(payload["event_params"]["responses_id"], "resp_123"); + assert_eq!(payload["event_params"]["turn_responses_call_index"], 2); + assert_eq!(payload["event_params"]["model"], "gpt-5"); + assert_eq!(payload["event_params"]["model_provider"], "openai"); + assert_eq!(payload["event_params"]["reasoning_effort"], "high"); + assert_eq!(payload["event_params"]["status"], "completed"); + assert_eq!(payload["event_params"]["error"], json!(null)); + assert_eq!(payload["event_params"]["started_at"], 100); + assert_eq!(payload["event_params"]["completed_at"], 102); + assert_eq!(payload["event_params"]["duration_ms"], 2345); + assert_eq!(payload["event_params"]["input_item_count"], 1); + assert_eq!(payload["event_params"]["output_item_count"], 1); + assert_eq!(payload["event_params"]["input_tokens"], 10); + assert_eq!(payload["event_params"]["cached_input_tokens"], 4); + assert_eq!(payload["event_params"]["output_tokens"], 20); + assert_eq!(payload["event_params"]["reasoning_output_tokens"], 3); + assert_eq!(payload["event_params"]["total_tokens"], 30); + assert_eq!(payload["event_params"]["items"][0]["item_phase"], "input"); + assert_eq!( + payload["event_params"]["items"][0]["response_item_type"], + "message" + ); + assert_eq!(payload["event_params"]["items"][0]["role"], "assistant"); + assert_eq!( + payload["event_params"]["items"][0]["message_phase"], + "final_answer" + ); + assert_eq!(payload["event_params"]["items"][0]["text_part_count"], 1); + assert_eq!(payload["event_params"]["items"][0]["image_part_count"], 0); + assert_eq!(payload["event_params"]["items"][1]["item_phase"], "output"); +} + #[test] fn app_used_dedupe_is_keyed_by_turn_and_connector() { let (sender, _receiver) = mpsc::channel(1); @@ -1191,6 +1309,88 @@ async fn guardian_review_event_ingests_custom_fact_with_optional_target_item() { assert_eq!(payload[0]["event_params"]["review_timeout_ms"], 90_000); } +#[tokio::test] +async fn responses_api_call_event_ingests_custom_fact() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut events, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ false, + /*include_token_usage*/ false, + ) + .await; + events.clear(); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::ResponsesApiCall(Box::new( + CodexResponsesApiCallFact { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + responses_id: Some("resp_456".to_string()), + turn_responses_call_index: 1, + status: CodexResponsesApiCallStatus::Completed, + error: None, + started_at: 100, + completed_at: Some(101), + duration_ms: Some(1200), + input_item_count: 1, + output_item_count: 1, + input_tokens: Some(123), + cached_input_tokens: Some(45), + output_tokens: Some(140), + reasoning_output_tokens: Some(13), + total_tokens: Some(321), + items: vec![ + sample_responses_api_item_metadata( + CodexResponsesApiItemPhase::Input, + /*item_index*/ 0, + ), + sample_responses_api_item_metadata( + CodexResponsesApiItemPhase::Output, + /*item_index*/ 0, + ), + ], + }, + ))), + &mut events, + ) + .await; + + let payload = serde_json::to_value(&events).expect("serialize events"); + assert_eq!(payload.as_array().expect("events array").len(), 1); + assert_eq!(payload[0]["event_type"], "codex_responses_api_call_event"); + assert_eq!(payload[0]["event_params"]["thread_id"], "thread-2"); + assert_eq!(payload[0]["event_params"]["turn_id"], "turn-2"); + assert_eq!(payload[0]["event_params"]["ephemeral"], false); + assert_eq!(payload[0]["event_params"]["thread_source"], "user"); + assert_eq!(payload[0]["event_params"]["initialization_mode"], "new"); + assert_eq!( + payload[0]["event_params"]["app_server_client"]["product_client_id"], + "codex-tui" + ); + assert_eq!(payload[0]["event_params"]["runtime"]["runtime_os"], "macos"); + assert_eq!(payload[0]["event_params"]["responses_id"], "resp_456"); + assert_eq!(payload[0]["event_params"]["turn_responses_call_index"], 1); + assert_eq!(payload[0]["event_params"]["model"], "gpt-5"); + assert_eq!(payload[0]["event_params"]["model_provider"], "openai"); + assert_eq!(payload[0]["event_params"]["reasoning_effort"], json!(null)); + assert_eq!(payload[0]["event_params"]["status"], "completed"); + assert_eq!(payload[0]["event_params"]["input_tokens"], 123); + assert_eq!( + payload[0]["event_params"]["items"][0]["item_phase"], + "input" + ); + assert_eq!( + payload[0]["event_params"]["items"][1]["item_phase"], + "output" + ); +} + #[test] fn subagent_thread_started_review_serializes_expected_shape() { let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request( diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 9cbd97f923..e0de1975dd 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -23,6 +23,7 @@ use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; +use crate::response_items::response_items_metadata; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; @@ -217,6 +218,45 @@ impl AnalyticsEventsClient { ))); } + pub fn track_responses_api_call( + &self, + tracking: TrackEventsContext, + input: CodexResponsesApiCallInput, + ) { + let mut items = + response_items_metadata(CodexResponsesApiItemPhase::Input, &input.input_items); + items.extend(response_items_metadata( + CodexResponsesApiItemPhase::Output, + &input.output_items, + )); + let token_usage = input.token_usage; + let event = CodexResponsesApiCallFact { + thread_id: tracking.thread_id, + turn_id: tracking.turn_id, + responses_id: input.responses_id, + turn_responses_call_index: input.turn_responses_call_index, + status: input.status, + error: input.error, + started_at: input.started_at, + completed_at: input.completed_at, + duration_ms: input.duration_ms, + input_item_count: input.input_items.len(), + output_item_count: input.output_items.len(), + input_tokens: token_usage.as_ref().map(|usage| usage.input_tokens), + cached_input_tokens: token_usage.as_ref().map(|usage| usage.cached_input_tokens), + output_tokens: token_usage.as_ref().map(|usage| usage.output_tokens), + reasoning_output_tokens: token_usage + .as_ref() + .map(|usage| usage.reasoning_output_tokens), + total_tokens: token_usage.as_ref().map(|usage| usage.total_tokens), + items, + }; + + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::ResponsesApiCall(Box::new(event)), + )); + } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 8a1b7d2046..f4110f78b2 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -57,7 +57,6 @@ pub(crate) enum TrackEventRequest { AppUsed(CodexAppUsedEventRequest), HookRun(CodexHookRunEventRequest), Compaction(Box), - #[allow(dead_code)] ResponsesApiCall(Box), TurnEvent(Box), TurnSteer(CodexTurnSteerEventRequest), diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 04408c4c2e..5f8fb34317 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -381,6 +381,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), + ResponsesApiCall(Box), GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 965b83c953..4d70242d09 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -6,6 +6,8 @@ use crate::events::CodexCompactionEventRequest; use crate::events::CodexHookRunEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; +use crate::events::CodexResponsesApiCallEventParams; +use crate::events::CodexResponsesApiCallEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventParams; use crate::events::CodexTurnEventRequest; @@ -203,6 +205,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::Compaction(input) => { self.ingest_compaction(*input, out); } + CustomAnalyticsFact::ResponsesApiCall(input) => { + self.ingest_responses_api_call(*input, out); + } CustomAnalyticsFact::GuardianReview(input) => { self.ingest_guardian_review(*input, out); } @@ -740,6 +745,91 @@ impl AnalyticsReducer { ))); } + fn ingest_responses_api_call( + &mut self, + input: CodexResponsesApiCallFact, + out: &mut Vec, + ) { + let Some(connection_id) = self.thread_connections.get(&input.thread_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing thread connection metadata" + ); + return; + }; + let Some(connection_state) = self.connections.get(connection_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + connection_id, + "dropping responses api call analytics event: missing connection metadata" + ); + return; + }; + let Some(thread_metadata) = self.thread_metadata.get(&input.thread_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing thread lifecycle metadata" + ); + return; + }; + let Some(turn_state) = self.turns.get(&input.turn_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing turn metadata" + ); + return; + }; + let Some(resolved_config) = turn_state.resolved_config.as_ref() else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing turn resolved config" + ); + return; + }; + + out.push(TrackEventRequest::ResponsesApiCall(Box::new( + CodexResponsesApiCallEventRequest { + event_type: "codex_responses_api_call_event", + event_params: CodexResponsesApiCallEventParams { + thread_id: input.thread_id, + turn_id: input.turn_id, + ephemeral: resolved_config.ephemeral, + thread_source: thread_metadata.thread_source.map(str::to_string), + initialization_mode: thread_metadata.initialization_mode, + subagent_source: thread_metadata.subagent_source.clone(), + parent_thread_id: thread_metadata.parent_thread_id.clone(), + app_server_client: connection_state.app_server_client.clone(), + runtime: connection_state.runtime.clone(), + responses_id: input.responses_id, + turn_responses_call_index: input.turn_responses_call_index, + model: Some(resolved_config.model.clone()), + model_provider: Some(resolved_config.model_provider.clone()), + reasoning_effort: resolved_config + .reasoning_effort + .map(|value| value.to_string()), + status: input.status, + error: input.error, + started_at: input.started_at, + completed_at: input.completed_at, + duration_ms: input.duration_ms, + input_item_count: input.input_item_count, + output_item_count: input.output_item_count, + input_tokens: input.input_tokens, + cached_input_tokens: input.cached_input_tokens, + output_tokens: input.output_tokens, + reasoning_output_tokens: input.reasoning_output_tokens, + total_tokens: input.total_tokens, + items: input.items, + }, + }, + ))); + } + fn ingest_turn_steer_response( &mut self, connection_id: u64,