From ce5991d84492f43e536543d07fd76ff2851cf542 Mon Sep 17 00:00:00 2001 From: Roy Han Date: Wed, 25 Mar 2026 18:54:44 -0700 Subject: [PATCH 1/2] expand schema --- codex-rs/analytics/src/analytics_client.rs | 88 +++++++++++++++++-- .../analytics/src/analytics_client_tests.rs | 46 +++++++++- codex-rs/analytics/src/lib.rs | 2 + codex-rs/core/src/codex.rs | 20 +++++ 4 files changed, 148 insertions(+), 8 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client.rs b/codex-rs/analytics/src/analytics_client.rs index 4d97992a53..4d62186753 100644 --- a/codex-rs/analytics/src/analytics_client.rs +++ b/codex-rs/analytics/src/analytics_client.rs @@ -11,6 +11,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::ServiceTier; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::CodexErrorInfo; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; @@ -36,6 +37,7 @@ pub struct TrackEventsContext { #[derive(Clone)] pub struct CodexTurnEvent { + pub submission_type: Option, pub model_provider: String, pub sandbox_policy: SandboxPolicy, pub reasoning_effort: Option, @@ -48,6 +50,40 @@ pub struct CodexTurnEvent { pub personality: Option, pub num_input_images: usize, pub is_first_turn: bool, + pub status: Option, + pub turn_error: Option, + pub steer_count: Option, + pub total_tool_call_count: Option, + pub shell_command_count: Option, + pub file_change_count: Option, + pub mcp_tool_call_count: Option, + pub dynamic_tool_call_count: Option, + pub subagent_tool_call_count: Option, + pub web_search_count: Option, + pub image_generation_count: Option, + pub input_tokens: Option, + pub cached_input_tokens: Option, + pub output_tokens: Option, + pub reasoning_output_tokens: Option, + pub total_tokens: Option, + pub duration_ms: Option, + pub started_at: Option, + pub completed_at: Option, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, } #[derive(Clone)] @@ -357,7 +393,7 @@ enum TrackEventRequest { CodexThreadInitialized(CodexThreadInitializedEvent), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), - TurnEvent(CodexTurnEventRequest), + TurnEvent(Box), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -430,6 +466,7 @@ struct CodexTurnEventParams { thread_id: String, turn_id: String, product_client_id: Option, + submission_type: Option, model: Option, model_provider: String, sandbox_policy: Option<&'static str>, @@ -443,6 +480,25 @@ struct CodexTurnEventParams { personality: Option, num_input_images: usize, is_first_turn: bool, + status: Option, + turn_error: Option, + steer_count: Option, + total_tool_call_count: Option, + shell_command_count: Option, + file_change_count: Option, + mcp_tool_call_count: Option, + dynamic_tool_call_count: Option, + subagent_tool_call_count: Option, + web_search_count: Option, + image_generation_count: Option, + input_tokens: Option, + cached_input_tokens: Option, + output_tokens: Option, + reasoning_output_tokens: Option, + total_tokens: Option, + duration_ms: Option, + started_at: Option, + completed_at: Option, } #[derive(Serialize)] @@ -531,10 +587,12 @@ impl AnalyticsReducer { tracking, turn_event, } = input; - out.push(TrackEventRequest::TurnEvent(CodexTurnEventRequest { - event_type: "codex_turn_event", - event_params: codex_turn_event_params(&tracking, turn_event), - })); + out.push(TrackEventRequest::TurnEvent(Box::new( + CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: codex_turn_event_params(&tracking, turn_event), + }, + ))); } async fn ingest_skill_invoked( @@ -660,6 +718,7 @@ fn codex_turn_event_params( thread_id: tracking.thread_id.clone(), turn_id: tracking.turn_id.clone(), product_client_id: Some(originator().value), + submission_type: turn_event.submission_type, model: Some(tracking.model_slug.clone()), model_provider: turn_event.model_provider, sandbox_policy: Some(sandbox_policy_mode(&turn_event.sandbox_policy)), @@ -676,6 +735,25 @@ fn codex_turn_event_params( personality: personality_mode(turn_event.personality), num_input_images: turn_event.num_input_images, is_first_turn: turn_event.is_first_turn, + status: turn_event.status, + turn_error: turn_event.turn_error, + steer_count: turn_event.steer_count, + total_tool_call_count: turn_event.total_tool_call_count, + shell_command_count: turn_event.shell_command_count, + file_change_count: turn_event.file_change_count, + mcp_tool_call_count: turn_event.mcp_tool_call_count, + dynamic_tool_call_count: turn_event.dynamic_tool_call_count, + subagent_tool_call_count: turn_event.subagent_tool_call_count, + web_search_count: turn_event.web_search_count, + image_generation_count: turn_event.image_generation_count, + input_tokens: turn_event.input_tokens, + cached_input_tokens: turn_event.cached_input_tokens, + output_tokens: turn_event.output_tokens, + reasoning_output_tokens: turn_event.reasoning_output_tokens, + total_tokens: turn_event.total_tokens, + duration_ms: turn_event.duration_ms, + started_at: turn_event.started_at, + completed_at: turn_event.completed_at, } } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 1d6f14edee..6a391042f0 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -210,11 +210,12 @@ fn turn_event_serializes_expected_shape() { thread_id: "thread-2".to_string(), turn_id: "turn-2".to_string(), }; - let event = TrackEventRequest::TurnEvent(CodexTurnEventRequest { + let event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest { event_type: "codex_turn_event", event_params: codex_turn_event_params( &tracking, CodexTurnEvent { + submission_type: None, model_provider: "openai".to_string(), sandbox_policy: SandboxPolicy::new_read_only_policy(), reasoning_effort: Some(ReasoningEffort::High), @@ -227,9 +228,28 @@ fn turn_event_serializes_expected_shape() { personality: Some(Personality::Pragmatic), num_input_images: 2, is_first_turn: true, + status: None, + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, + duration_ms: None, + started_at: None, + completed_at: None, }, ), - }); + })); let payload = serde_json::to_value(&event).expect("serialize turn event"); @@ -241,6 +261,7 @@ fn turn_event_serializes_expected_shape() { "thread_id": "thread-2", "turn_id": "turn-2", "product_client_id": originator().value, + "submission_type": null, "model": "gpt-5", "model_provider": "openai", "sandbox_policy": "read_only", @@ -253,7 +274,26 @@ fn turn_event_serializes_expected_shape() { "collaboration_mode": "plan", "personality": "pragmatic", "num_input_images": 2, - "is_first_turn": true + "is_first_turn": true, + "status": null, + "turn_error": null, + "steer_count": null, + "total_tool_call_count": null, + "shell_command_count": null, + "file_change_count": null, + "mcp_tool_call_count": null, + "dynamic_tool_call_count": null, + "subagent_tool_call_count": null, + "web_search_count": null, + "image_generation_count": null, + "input_tokens": null, + "cached_input_tokens": null, + "output_tokens": null, + "reasoning_output_tokens": null, + "total_tokens": null, + "duration_ms": null, + "started_at": null, + "completed_at": null } }) ); diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index ba8dc31c0a..5956ad1c7f 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -17,4 +17,6 @@ pub use analytics_client::PluginUsedInput; pub use analytics_client::SkillInvocation; pub use analytics_client::SkillInvokedInput; pub use analytics_client::TrackEventsContext; +pub use analytics_client::TurnStatus; +pub use analytics_client::TurnSubmissionType; pub use analytics_client::build_track_events_context; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 2bcab0921e..d863c06997 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -6149,6 +6149,7 @@ pub(crate) async fn run_turn( sess.services.analytics_events_client.track_turn_event( tracking, CodexTurnEvent { + submission_type: None, model_provider: turn_context.config.model_provider_id.clone(), sandbox_policy: turn_context.sandbox_policy.get().clone(), reasoning_effort: turn_context.reasoning_effort, @@ -6166,6 +6167,25 @@ pub(crate) async fn run_turn( }) .count(), is_first_turn, + status: None, + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, + duration_ms: None, + started_at: None, + completed_at: None, }, ); } From 5a449cd2d04868956470b2a8b34631430d169067 Mon Sep 17 00:00:00 2001 From: Roy Han Date: Wed, 25 Mar 2026 23:13:22 -0700 Subject: [PATCH 2/2] turn start and complete --- codex-rs/analytics/src/analytics_client.rs | 54 ++++++++-- .../analytics/src/analytics_client_tests.rs | 75 ++++++++++++++ codex-rs/analytics/src/lib.rs | 2 + codex-rs/core/src/codex.rs | 98 ++++++++++--------- 4 files changed, 174 insertions(+), 55 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client.rs b/codex-rs/analytics/src/analytics_client.rs index 4d62186753..224cf926c4 100644 --- a/codex-rs/analytics/src/analytics_client.rs +++ b/codex-rs/analytics/src/analytics_client.rs @@ -147,7 +147,8 @@ pub struct AppInvocation { pub enum AnalyticsInput { CodexThreadInitialized(CodexThreadInitializedInput), - TurnEvent(TurnEventInput), + TurnStarted(TurnStartedInput), + TurnCompleted(TurnCompletedInput), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), @@ -155,10 +156,15 @@ pub enum AnalyticsInput { PluginStateChanged(PluginStateChangedInput), } -pub struct TurnEventInput { +pub struct TurnStartedInput { pub tracking: TrackEventsContext, pub turn_event: CodexTurnEvent, } + +pub struct TurnCompletedInput { + pub turn_id: String, +} + pub struct SkillInvokedInput { pub tracking: TrackEventsContext, pub invocations: Vec, @@ -195,12 +201,17 @@ pub enum PluginState { #[derive(Default)] pub struct AnalyticsReducer { threads: HashMap, + turns: HashMap, } struct ThreadState { _initialized_input: CodexThreadInitializedInput, } +struct TurnState { + started_input: TurnStartedInput, +} + #[derive(Clone)] pub(crate) struct AnalyticsEventsQueue { sender: mpsc::Sender, @@ -326,13 +337,19 @@ impl AnalyticsEventsClient { })); } - pub fn track_turn_event(&self, tracking: TrackEventsContext, turn_event: CodexTurnEvent) { - self.record(AnalyticsInput::TurnEvent(TurnEventInput { + pub fn track_turn_started(&self, tracking: TrackEventsContext, turn_event: CodexTurnEvent) { + self.record(AnalyticsInput::TurnStarted(TurnStartedInput { tracking, turn_event, })); } + pub fn track_turn_completed(&self, turn_id: String) { + self.record(AnalyticsInput::TurnCompleted(TurnCompletedInput { + turn_id, + })); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record(AnalyticsInput::PluginStateChanged( PluginStateChangedInput { @@ -545,8 +562,11 @@ impl AnalyticsReducer { AnalyticsInput::CodexThreadInitialized(input) => { self.ingest_thread_initialized(input, out); } - AnalyticsInput::TurnEvent(input) => { - self.ingest_turn_event(input, out); + AnalyticsInput::TurnStarted(input) => { + self.ingest_turn_started(input); + } + AnalyticsInput::TurnCompleted(input) => { + self.ingest_turn_completed(input, out); } AnalyticsInput::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; @@ -582,11 +602,27 @@ impl AnalyticsReducer { )); } - fn ingest_turn_event(&mut self, input: TurnEventInput, out: &mut Vec) { - let TurnEventInput { + fn ingest_turn_started(&mut self, input: TurnStartedInput) { + self.turns.insert( + input.tracking.turn_id.clone(), + TurnState { + started_input: input, + }, + ); + } + + fn ingest_turn_completed( + &mut self, + input: TurnCompletedInput, + out: &mut Vec, + ) { + let Some(TurnState { started_input }) = self.turns.remove(&input.turn_id) else { + return; + }; + let TurnStartedInput { tracking, turn_event, - } = input; + } = started_input; out.push(TrackEventRequest::TurnEvent(Box::new( CodexTurnEventRequest { event_type: "codex_turn_event", diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 6a391042f0..8fd1b5eada 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1,4 +1,6 @@ use super::AnalyticsEventsQueue; +use super::AnalyticsInput; +use super::AnalyticsReducer; use super::AppInvocation; use super::CodexAppMentionedEventRequest; use super::CodexAppUsedEventRequest; @@ -12,6 +14,8 @@ use super::InitializationMode; use super::InvocationType; use super::TrackEventRequest; use super::TrackEventsContext; +use super::TurnCompletedInput; +use super::TurnStartedInput; use super::codex_app_metadata; use super::codex_plugin_metadata; use super::codex_plugin_used_metadata; @@ -299,6 +303,77 @@ fn turn_event_serializes_expected_shape() { ); } +#[tokio::test] +async fn turn_started_then_completed_emits_turn_event() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }; + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsInput::TurnStarted(TurnStartedInput { + tracking: tracking.clone(), + turn_event: CodexTurnEvent { + submission_type: None, + model_provider: "openai".to_string(), + sandbox_policy: SandboxPolicy::new_read_only_policy(), + reasoning_effort: Some(ReasoningEffort::High), + reasoning_summary: Some(ReasoningSummary::Detailed), + service_tier: Some(ServiceTier::Flex), + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: ApprovalsReviewer::GuardianSubagent, + sandbox_network_access: true, + collaboration_mode: ModeKind::Plan, + personality: Some(Personality::Pragmatic), + num_input_images: 2, + is_first_turn: true, + status: None, + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, + duration_ms: None, + started_at: None, + completed_at: None, + }, + }), + &mut out, + ) + .await; + + assert!(out.is_empty()); + + reducer + .ingest( + AnalyticsInput::TurnCompleted(TurnCompletedInput { + turn_id: tracking.turn_id.clone(), + }), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_type"], json!("codex_turn_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); +} + #[test] fn thread_initialized_event_serializes_expected_shape() { let event = TrackEventRequest::CodexThreadInitialized(codex_thread_initialized_event_request( diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 5956ad1c7f..9303aeaf90 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -17,6 +17,8 @@ pub use analytics_client::PluginUsedInput; pub use analytics_client::SkillInvocation; pub use analytics_client::SkillInvokedInput; pub use analytics_client::TrackEventsContext; +pub use analytics_client::TurnCompletedInput; +pub use analytics_client::TurnStartedInput; pub use analytics_client::TurnStatus; pub use analytics_client::TurnSubmissionType; pub use analytics_client::build_track_events_context; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index d863c06997..b76a016522 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -5859,6 +5859,55 @@ pub(crate) async fn run_turn( .await; } + if !input.is_empty() { + let is_first_turn = { + let mut state = sess.state.lock().await; + state.take_next_turn_is_first() + }; + sess.services.analytics_events_client.track_turn_started( + tracking.clone(), + CodexTurnEvent { + submission_type: None, + model_provider: turn_context.config.model_provider_id.clone(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + reasoning_effort: turn_context.reasoning_effort, + reasoning_summary: Some(turn_context.reasoning_summary), + service_tier: turn_context.config.service_tier, + approval_policy: turn_context.approval_policy.value(), + approvals_reviewer: turn_context.config.approvals_reviewer, + sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), + collaboration_mode: turn_context.collaboration_mode.mode, + personality: turn_context.personality, + num_input_images: input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + is_first_turn, + status: None, + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, + duration_ms: None, + started_at: None, + completed_at: None, + }, + ); + } + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) .await; @@ -6142,52 +6191,9 @@ pub(crate) async fn run_turn( } if !input.is_empty() { - let is_first_turn = { - let mut state = sess.state.lock().await; - state.take_next_turn_is_first() - }; - sess.services.analytics_events_client.track_turn_event( - tracking, - CodexTurnEvent { - submission_type: None, - model_provider: turn_context.config.model_provider_id.clone(), - sandbox_policy: turn_context.sandbox_policy.get().clone(), - reasoning_effort: turn_context.reasoning_effort, - reasoning_summary: Some(turn_context.reasoning_summary), - service_tier: turn_context.config.service_tier, - approval_policy: turn_context.approval_policy.value(), - approvals_reviewer: turn_context.config.approvals_reviewer, - sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), - collaboration_mode: turn_context.collaboration_mode.mode, - personality: turn_context.personality, - num_input_images: input - .iter() - .filter(|item| { - matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) - }) - .count(), - is_first_turn, - status: None, - turn_error: None, - steer_count: None, - total_tool_call_count: None, - shell_command_count: None, - file_change_count: None, - mcp_tool_call_count: None, - dynamic_tool_call_count: None, - subagent_tool_call_count: None, - web_search_count: None, - image_generation_count: None, - input_tokens: None, - cached_input_tokens: None, - output_tokens: None, - reasoning_output_tokens: None, - total_tokens: None, - duration_ms: None, - started_at: None, - completed_at: None, - }, - ); + sess.services + .analytics_events_client + .track_turn_completed(tracking.turn_id.clone()); } last_agent_message