From b1fbffeacfc4dba92cf8e9b255d7d0998dc89bb4 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Wed, 15 Apr 2026 17:23:07 -0700 Subject: [PATCH] analytics: reduce responses api call facts --- .../analytics/src/analytics_client_tests.rs | 199 ++++++++++++++++++ codex-rs/analytics/src/client.rs | 7 + codex-rs/analytics/src/facts.rs | 2 + codex-rs/analytics/src/reducer.rs | 91 ++++++++ 4 files changed, 299 insertions(+) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index eb46674156..2ea2ae4f62 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -6,6 +6,8 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexCompactionEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; +use crate::events::CodexResponsesApiCallEventParams; +use crate::events::CodexResponsesApiCallEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventRequest; use crate::events::ThreadInitializedEvent; @@ -45,6 +47,13 @@ use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; +use crate::responses_api::CodexResponseItemRole; +use crate::responses_api::CodexResponseItemType; +use crate::responses_api::CodexResponseMessagePhase; +use crate::responses_api::CodexResponsesApiCallFact; +use crate::responses_api::CodexResponsesApiCallStatus; +use crate::responses_api::CodexResponsesApiItemMetadata; +use crate::responses_api::CodexResponsesApiItemPhase; use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer; use codex_app_server_protocol::AskForApproval as AppServerAskForApproval; use codex_app_server_protocol::ClientInfo; @@ -259,6 +268,30 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag } } +fn sample_responses_api_item_metadata( + item_phase: CodexResponsesApiItemPhase, + item_index: usize, +) -> CodexResponsesApiItemMetadata { + CodexResponsesApiItemMetadata { + item_phase, + item_index, + response_item_type: CodexResponseItemType::Message, + role: Some(CodexResponseItemRole::Assistant), + status: Some("completed".to_string()), + message_phase: Some(CodexResponseMessagePhase::FinalAnswer), + call_id: None, + tool_name: None, + content_bytes: Some(12), + arguments_bytes: None, + output_bytes: None, + encrypted_content_bytes: None, + image_generation_revised_prompt_bytes: None, + image_generation_result_bytes: None, + text_part_count: Some(1), + image_part_count: Some(0), + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -738,6 +771,96 @@ fn compaction_event_serializes_expected_shape() { ); } +#[test] +fn responses_api_call_event_serializes_expected_shape() { + let event = TrackEventRequest::ResponsesApiCall(Box::new(CodexResponsesApiCallEventRequest { + event_type: "codex_responses_api_call_event", + event_params: CodexResponsesApiCallEventParams { + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + ephemeral: false, + thread_source: Some("user".to_string()), + initialization_mode: ThreadInitializationMode::New, + subagent_source: None, + parent_thread_id: None, + app_server_client: sample_app_server_client_metadata(), + runtime: sample_runtime_metadata(), + responses_id: Some("resp_123".to_string()), + turn_responses_call_index: 2, + model: Some("gpt-5".to_string()), + model_provider: Some("openai".to_string()), + reasoning_effort: Some("high".to_string()), + status: CodexResponsesApiCallStatus::Completed, + error: None, + started_at: 100, + completed_at: Some(102), + duration_ms: Some(2345), + input_item_count: 1, + output_item_count: 1, + input_tokens: Some(10), + cached_input_tokens: Some(4), + output_tokens: Some(20), + reasoning_output_tokens: Some(3), + total_tokens: Some(30), + items: vec![ + sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Input, 0), + sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Output, 0), + ], + }, + })); + + let payload = serde_json::to_value(&event).expect("serialize responses api call event"); + + assert_eq!(payload["event_type"], "codex_responses_api_call_event"); + assert_eq!(payload["event_params"]["thread_id"], "thread-1"); + assert_eq!(payload["event_params"]["turn_id"], "turn-1"); + assert_eq!(payload["event_params"]["ephemeral"], false); + assert_eq!(payload["event_params"]["thread_source"], "user"); + assert_eq!(payload["event_params"]["initialization_mode"], "new"); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + DEFAULT_ORIGINATOR + ); + assert_eq!( + payload["event_params"]["app_server_client"]["rpc_transport"], + "stdio" + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + "0.1.0" + ); + assert_eq!(payload["event_params"]["responses_id"], "resp_123"); + assert_eq!(payload["event_params"]["turn_responses_call_index"], 2); + assert_eq!(payload["event_params"]["model"], "gpt-5"); + assert_eq!(payload["event_params"]["model_provider"], "openai"); + assert_eq!(payload["event_params"]["reasoning_effort"], "high"); + assert_eq!(payload["event_params"]["status"], "completed"); + assert_eq!(payload["event_params"]["error"], json!(null)); + assert_eq!(payload["event_params"]["started_at"], 100); + assert_eq!(payload["event_params"]["completed_at"], 102); + assert_eq!(payload["event_params"]["duration_ms"], 2345); + assert_eq!(payload["event_params"]["input_item_count"], 1); + assert_eq!(payload["event_params"]["output_item_count"], 1); + assert_eq!(payload["event_params"]["input_tokens"], 10); + assert_eq!(payload["event_params"]["cached_input_tokens"], 4); + assert_eq!(payload["event_params"]["output_tokens"], 20); + assert_eq!(payload["event_params"]["reasoning_output_tokens"], 3); + assert_eq!(payload["event_params"]["total_tokens"], 30); + assert_eq!(payload["event_params"]["items"][0]["item_phase"], "input"); + assert_eq!( + payload["event_params"]["items"][0]["response_item_type"], + "message" + ); + assert_eq!(payload["event_params"]["items"][0]["role"], "assistant"); + assert_eq!( + payload["event_params"]["items"][0]["message_phase"], + "final_answer" + ); + assert_eq!(payload["event_params"]["items"][0]["text_part_count"], 1); + assert_eq!(payload["event_params"]["items"][0]["image_part_count"], 0); + assert_eq!(payload["event_params"]["items"][1]["item_phase"], "output"); +} + #[test] fn app_used_dedupe_is_keyed_by_turn_and_connector() { let (sender, _receiver) = mpsc::channel(1); @@ -1043,6 +1166,82 @@ async fn compaction_event_ingests_custom_fact() { assert_eq!(payload[0]["event_params"]["status"], "failed"); } +#[tokio::test] +async fn responses_api_call_event_ingests_custom_fact() { + let mut reducer = AnalyticsReducer::default(); + let mut events = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut events, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ false, + /*include_token_usage*/ false, + ) + .await; + events.clear(); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::ResponsesApiCall(Box::new( + CodexResponsesApiCallFact { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + responses_id: Some("resp_456".to_string()), + turn_responses_call_index: 1, + status: CodexResponsesApiCallStatus::Completed, + error: None, + started_at: 100, + completed_at: Some(101), + duration_ms: Some(1200), + input_item_count: 1, + output_item_count: 1, + input_tokens: Some(123), + cached_input_tokens: Some(45), + output_tokens: Some(140), + reasoning_output_tokens: Some(13), + total_tokens: Some(321), + items: vec![ + sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Input, 0), + sample_responses_api_item_metadata(CodexResponsesApiItemPhase::Output, 0), + ], + }, + ))), + &mut events, + ) + .await; + + let payload = serde_json::to_value(&events).expect("serialize events"); + assert_eq!(payload.as_array().expect("events array").len(), 1); + assert_eq!(payload[0]["event_type"], "codex_responses_api_call_event"); + assert_eq!(payload[0]["event_params"]["thread_id"], "thread-2"); + assert_eq!(payload[0]["event_params"]["turn_id"], "turn-2"); + assert_eq!(payload[0]["event_params"]["ephemeral"], false); + assert_eq!(payload[0]["event_params"]["thread_source"], "user"); + assert_eq!(payload[0]["event_params"]["initialization_mode"], "new"); + assert_eq!( + payload[0]["event_params"]["app_server_client"]["product_client_id"], + "codex-tui" + ); + assert_eq!(payload[0]["event_params"]["runtime"]["runtime_os"], "macos"); + assert_eq!(payload[0]["event_params"]["responses_id"], "resp_456"); + assert_eq!(payload[0]["event_params"]["turn_responses_call_index"], 1); + assert_eq!(payload[0]["event_params"]["model"], "gpt-5"); + assert_eq!(payload[0]["event_params"]["model_provider"], "openai"); + assert_eq!(payload[0]["event_params"]["reasoning_effort"], json!(null)); + assert_eq!(payload[0]["event_params"]["status"], "completed"); + assert_eq!(payload[0]["event_params"]["input_tokens"], 123); + assert_eq!( + payload[0]["event_params"]["items"][0]["item_phase"], + "input" + ); + assert_eq!( + payload[0]["event_params"]["items"][1]["item_phase"], + "output" + ); +} + #[test] fn subagent_thread_started_review_serializes_expected_shape() { let event = TrackEventRequest::ThreadInitialized(subagent_thread_started_event_request( diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 5ba60e3ef4..e00c966965 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -18,6 +18,7 @@ use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; +use crate::responses_api::CodexResponsesApiCallFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; @@ -206,6 +207,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_responses_api_call(&self, event: CodexResponsesApiCallFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::ResponsesApiCall(Box::new(event)), + )); + } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 5590fbfa6d..e94e3c9a75 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -1,6 +1,7 @@ use crate::events::AppServerRpcTransport; use crate::events::CodexRuntimeMetadata; use crate::events::GuardianReviewEventParams; +use crate::responses_api::CodexResponsesApiCallFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; @@ -292,6 +293,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), + ResponsesApiCall(Box), GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index a5e5165bd2..fdc1f2f3eb 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -5,6 +5,8 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexCompactionEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; +use crate::events::CodexResponsesApiCallEventParams; +use crate::events::CodexResponsesApiCallEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventParams; use crate::events::CodexTurnEventRequest; @@ -44,6 +46,7 @@ use crate::facts::TurnSteerRejectionReason; use crate::facts::TurnSteerResult; use crate::facts::TurnTokenUsageFact; use crate::now_unix_seconds; +use crate::responses_api::CodexResponsesApiCallFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::CodexErrorInfo; @@ -199,6 +202,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::Compaction(input) => { self.ingest_compaction(*input, out); } + CustomAnalyticsFact::ResponsesApiCall(input) => { + self.ingest_responses_api_call(*input, out); + } CustomAnalyticsFact::GuardianReview(input) => { self.ingest_guardian_review(*input, out); } @@ -725,6 +731,91 @@ impl AnalyticsReducer { ))); } + fn ingest_responses_api_call( + &mut self, + input: CodexResponsesApiCallFact, + out: &mut Vec, + ) { + let Some(connection_id) = self.thread_connections.get(&input.thread_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing thread connection metadata" + ); + return; + }; + let Some(connection_state) = self.connections.get(connection_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + connection_id, + "dropping responses api call analytics event: missing connection metadata" + ); + return; + }; + let Some(thread_metadata) = self.thread_metadata.get(&input.thread_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing thread lifecycle metadata" + ); + return; + }; + let Some(turn_state) = self.turns.get(&input.turn_id) else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing turn metadata" + ); + return; + }; + let Some(resolved_config) = turn_state.resolved_config.as_ref() else { + tracing::warn!( + thread_id = %input.thread_id, + turn_id = %input.turn_id, + "dropping responses api call analytics event: missing turn resolved config" + ); + return; + }; + + out.push(TrackEventRequest::ResponsesApiCall(Box::new( + CodexResponsesApiCallEventRequest { + event_type: "codex_responses_api_call_event", + event_params: CodexResponsesApiCallEventParams { + thread_id: input.thread_id, + turn_id: input.turn_id, + ephemeral: resolved_config.ephemeral, + thread_source: thread_metadata.thread_source.map(str::to_string), + initialization_mode: thread_metadata.initialization_mode, + subagent_source: thread_metadata.subagent_source.clone(), + parent_thread_id: thread_metadata.parent_thread_id.clone(), + app_server_client: connection_state.app_server_client.clone(), + runtime: connection_state.runtime.clone(), + responses_id: input.responses_id, + turn_responses_call_index: input.turn_responses_call_index, + model: Some(resolved_config.model.clone()), + model_provider: Some(resolved_config.model_provider.clone()), + reasoning_effort: resolved_config + .reasoning_effort + .map(|value| value.to_string()), + status: input.status, + error: input.error, + started_at: input.started_at, + completed_at: input.completed_at, + duration_ms: input.duration_ms, + input_item_count: input.input_item_count, + output_item_count: input.output_item_count, + input_tokens: input.input_tokens, + cached_input_tokens: input.cached_input_tokens, + output_tokens: input.output_tokens, + reasoning_output_tokens: input.reasoning_output_tokens, + total_tokens: input.total_tokens, + items: input.items, + }, + }, + ))); + } + fn ingest_turn_steer_response( &mut self, connection_id: u64,