From 852cf6a2a52c22ece86131833432a6d275ab6ceb Mon Sep 17 00:00:00 2001 From: Roy Han Date: Mon, 23 Mar 2026 18:46:43 -0700 Subject: [PATCH] turn level metadata --- codex-rs/core/src/analytics_client.rs | 128 ++++++++++++++++++++ codex-rs/core/src/analytics_client_tests.rs | 50 ++++++++ codex-rs/core/src/codex.rs | 13 ++ codex-rs/core/tests/suite/items.rs | 96 +++++++++++++++ 4 files changed, 287 insertions(+) diff --git a/codex-rs/core/src/analytics_client.rs b/codex-rs/core/src/analytics_client.rs index bfb90bc4da..5862281126 100644 --- a/codex-rs/core/src/analytics_client.rs +++ b/codex-rs/core/src/analytics_client.rs @@ -4,6 +4,11 @@ use crate::default_client::create_client; use crate::git_info::collect_git_info; use crate::git_info::get_git_repo_root; use crate::plugins::PluginTelemetryMetadata; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SkillScope; use serde::Serialize; use sha1::Digest; @@ -23,6 +28,15 @@ pub(crate) struct TrackEventsContext { pub(crate) turn_id: String, } +#[derive(Clone)] +pub(crate) struct TurnMetadata { + pub(crate) sandbox_policy: SandboxPolicy, + pub(crate) effort: Option, + pub(crate) summary: ReasoningSummary, + pub(crate) service_tier: Option, + pub(crate) collaboration_mode: ModeKind, +} + pub(crate) fn build_track_events_context( model_slug: String, thread_id: String, @@ -84,6 +98,9 @@ impl AnalyticsEventsQueue { TrackEventsJob::AppUsed(job) => { send_track_app_used(&auth_manager, job).await; } + TrackEventsJob::TurnMetadata(job) => { + send_track_turn_metadata(&auth_manager, job).await; + } TrackEventsJob::PluginUsed(job) => { send_track_plugin_used(&auth_manager, job).await; } @@ -197,6 +214,19 @@ impl AnalyticsEventsClient { ); } + pub(crate) fn track_turn_metadata( + &self, + tracking: TrackEventsContext, + turn_metadata: TurnMetadata, + ) { + track_turn_metadata( + &self.queue, + Arc::clone(&self.config), + Some(tracking), + turn_metadata, + ); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { track_plugin_management( &self.queue, @@ -238,6 +268,7 @@ enum TrackEventsJob { SkillInvocations(TrackSkillInvocationsJob), AppMentioned(TrackAppMentionedJob), AppUsed(TrackAppUsedJob), + TurnMetadata(TrackTurnMetadataJob), PluginUsed(TrackPluginUsedJob), PluginInstalled(TrackPluginManagementJob), PluginUninstalled(TrackPluginManagementJob), @@ -263,6 +294,12 @@ struct TrackAppUsedJob { app: AppInvocation, } +struct TrackTurnMetadataJob { + config: Arc, + tracking: TrackEventsContext, + turn_metadata: TurnMetadata, +} + struct TrackPluginUsedJob { config: Arc, tracking: TrackEventsContext, @@ -297,6 +334,7 @@ enum TrackEventRequest { SkillInvocation(SkillInvocationEventRequest), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), + TurnMetadata(CodexTurnMetadataEventRequest), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -345,6 +383,25 @@ struct CodexAppUsedEventRequest { event_params: CodexAppMetadata, } +#[derive(Serialize)] +struct CodexTurnMetadata { + thread_id: Option, + turn_id: Option, + product_client_id: Option, + model_slug: Option, + sandbox_policy: Option<&'static str>, + effort: Option, + summary: Option, + service_tier: Option, + collaboration_mode: Option<&'static str>, +} + +#[derive(Serialize)] +struct CodexTurnMetadataEventRequest { + event_type: &'static str, + event_params: CodexTurnMetadata, +} + #[derive(Serialize)] struct CodexPluginMetadata { plugin_id: Option, @@ -446,6 +503,26 @@ pub(crate) fn track_app_used( queue.try_send(job); } +pub(crate) fn track_turn_metadata( + queue: &AnalyticsEventsQueue, + config: Arc, + tracking: Option, + turn_metadata: TurnMetadata, +) { + if config.analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + let job = TrackEventsJob::TurnMetadata(TrackTurnMetadataJob { + config, + tracking, + turn_metadata, + }); + queue.try_send(job); +} + pub(crate) fn track_plugin_used( queue: &AnalyticsEventsQueue, config: Arc, @@ -571,6 +648,22 @@ async fn send_track_app_used(auth_manager: &AuthManager, job: TrackAppUsedJob) { send_track_events(auth_manager, config, events).await; } +async fn send_track_turn_metadata(auth_manager: &AuthManager, job: TrackTurnMetadataJob) { + let TrackTurnMetadataJob { + config, + tracking, + turn_metadata, + } = job; + let events = vec![TrackEventRequest::TurnMetadata( + CodexTurnMetadataEventRequest { + event_type: "codex_turn_metadata", + event_params: codex_turn_metadata(&tracking, turn_metadata), + }, + )]; + + send_track_events(auth_manager, config, events).await; +} + async fn send_track_plugin_used(auth_manager: &AuthManager, job: TrackPluginUsedJob) { let TrackPluginUsedJob { config, @@ -635,6 +728,41 @@ fn codex_app_metadata(tracking: &TrackEventsContext, app: AppInvocation) -> Code } } +fn codex_turn_metadata( + tracking: &TrackEventsContext, + turn_metadata: TurnMetadata, +) -> CodexTurnMetadata { + CodexTurnMetadata { + thread_id: Some(tracking.thread_id.clone()), + turn_id: Some(tracking.turn_id.clone()), + product_client_id: Some(crate::default_client::originator().value), + model_slug: Some(tracking.model_slug.clone()), + sandbox_policy: Some(sandbox_policy_mode(&turn_metadata.sandbox_policy)), + effort: turn_metadata.effort.map(|value| value.to_string()), + summary: Some(turn_metadata.summary.to_string()), + service_tier: turn_metadata.service_tier.map(|value| value.to_string()), + collaboration_mode: Some(collaboration_mode_mode(turn_metadata.collaboration_mode)), + } +} + +fn sandbox_policy_mode(sandbox_policy: &SandboxPolicy) -> &'static str { + match sandbox_policy { + SandboxPolicy::DangerFullAccess => "full_access", + SandboxPolicy::ReadOnly { .. } => "read_only", + SandboxPolicy::WorkspaceWrite { .. } => "workspace_write", + SandboxPolicy::ExternalSandbox { .. } => "external_sandbox", + } +} + +fn collaboration_mode_mode(mode: ModeKind) -> &'static str { + match mode { + ModeKind::Plan => "plan", + ModeKind::Default => "default", + ModeKind::PairProgramming => "pair_programming", + ModeKind::Execute => "execute", + } +} + fn codex_plugin_metadata(plugin: PluginTelemetryMetadata) -> CodexPluginMetadata { let capability_summary = plugin.capability_summary; CodexPluginMetadata { diff --git a/codex-rs/core/src/analytics_client_tests.rs b/codex-rs/core/src/analytics_client_tests.rs index a038aae047..e78b4538fd 100644 --- a/codex-rs/core/src/analytics_client_tests.rs +++ b/codex-rs/core/src/analytics_client_tests.rs @@ -4,17 +4,25 @@ use super::CodexAppMentionedEventRequest; use super::CodexAppUsedEventRequest; use super::CodexPluginEventRequest; use super::CodexPluginUsedEventRequest; +use super::CodexTurnMetadataEventRequest; use super::InvocationType; use super::TrackEventRequest; use super::TrackEventsContext; +use super::TurnMetadata; use super::codex_app_metadata; use super::codex_plugin_metadata; use super::codex_plugin_used_metadata; +use super::codex_turn_metadata; use super::normalize_path_for_skill_id; use crate::plugins::AppConnectorId; use crate::plugins::PluginCapabilitySummary; use crate::plugins::PluginId; use crate::plugins::PluginTelemetryMetadata; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::SandboxPolicy; use pretty_assertions::assert_eq; use serde_json::json; use std::collections::HashSet; @@ -185,6 +193,48 @@ fn app_used_dedupe_is_keyed_by_turn_and_connector() { assert_eq!(queue.should_enqueue_app_used(&turn_2, &app), true); } +#[test] +fn turn_metadata_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }; + let event = TrackEventRequest::TurnMetadata(CodexTurnMetadataEventRequest { + event_type: "codex_turn_metadata", + event_params: codex_turn_metadata( + &tracking, + TurnMetadata { + sandbox_policy: SandboxPolicy::new_read_only_policy(), + effort: Some(ReasoningEffort::High), + summary: ReasoningSummary::Detailed, + service_tier: Some(ServiceTier::Flex), + collaboration_mode: ModeKind::Plan, + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize turn metadata event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_turn_metadata", + "event_params": { + "thread_id": "thread-2", + "turn_id": "turn-2", + "product_client_id": crate::default_client::originator().value, + "model_slug": "gpt-5", + "sandbox_policy": "read_only", + "effort": "high", + "summary": "detailed", + "service_tier": "flex", + "collaboration_mode": "plan" + } + }) + ); +} + #[test] fn plugin_used_event_serializes_expected_shape() { let tracking = TrackEventsContext { diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 5e428362f2..6f7eaa122b 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -15,6 +15,7 @@ use crate::agent::agent_status_from_event; use crate::analytics_client::AnalyticsEventsClient; use crate::analytics_client::AppInvocation; use crate::analytics_client::InvocationType; +use crate::analytics_client::TurnMetadata; use crate::analytics_client::build_track_events_context; use crate::apps::render_apps_section; use crate::auth_env_telemetry::collect_auth_env_telemetry; @@ -5660,6 +5661,18 @@ pub(crate) async fn run_turn( .await; user_prompt_submit_outcome.additional_contexts }; + if !input.is_empty() { + sess.services.analytics_events_client.track_turn_metadata( + tracking.clone(), + TurnMetadata { + sandbox_policy: turn_context.sandbox_policy.get().clone(), + effort: turn_context.reasoning_effort, + summary: turn_context.reasoning_summary, + service_tier: turn_context.config.service_tier, + collaboration_mode: turn_context.collaboration_mode.mode, + }, + ); + } sess.services .analytics_events_client .track_app_mentioned(tracking.clone(), mentioned_app_invocations); diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index f949cd4b97..d6a6c50bcc 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -1,16 +1,22 @@ #![cfg(not(target_os = "windows"))] use anyhow::Ok; +use codex_core::CodexAuth; use codex_protocol::config_types::CollaborationMode; use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; use codex_protocol::config_types::Settings; use codex_protocol::items::AgentMessageContent; use codex_protocol::items::TurnItem; use codex_protocol::models::WebSearchAction; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::ItemCompletedEvent; use codex_protocol::protocol::ItemStartedEvent; use codex_protocol::protocol::Op; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::user_input::ByteRange; use codex_protocol::user_input::TextElement; use codex_protocol::user_input::UserInput; @@ -35,8 +41,11 @@ use core_test_support::test_codex::test_codex; use core_test_support::wait_for_event; use core_test_support::wait_for_event_match; use pretty_assertions::assert_eq; +use serde_json::Value; use std::path::Path; use std::path::PathBuf; +use std::time::Duration; +use std::time::Instant; fn image_generation_artifact_path(codex_home: &Path, session_id: &str, call_id: &str) -> PathBuf { fn sanitize(value: &str) -> String { @@ -120,6 +129,93 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn user_turn_tracks_turn_metadata_analytics() -> anyhow::Result<()> { + skip_if_no_network!(Ok(())); + + let server = start_mock_server().await; + mount_sse_once( + &server, + sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]), + ) + .await; + + let mut builder = test_codex().with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing()); + let TestCodex { + codex, + session_configured, + config, + .. + } = builder.build(&server).await?; + + codex + .submit(Op::UserTurn { + items: vec![UserInput::Text { + text: "hello turn metadata analytics".into(), + text_elements: Vec::new(), + }], + final_output_json_schema: None, + cwd: config.cwd.clone(), + approval_policy: AskForApproval::Never, + approvals_reviewer: None, + sandbox_policy: SandboxPolicy::new_read_only_policy(), + model: session_configured.model.clone(), + effort: Some(ReasoningEffort::High), + summary: Some(ReasoningSummary::Detailed), + service_tier: Some(Some(ServiceTier::Flex)), + collaboration_mode: Some(CollaborationMode { + mode: ModeKind::Plan, + settings: Settings::default(), + }), + personality: None, + }) + .await?; + + wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await; + + let deadline = Instant::now() + Duration::from_secs(10); + let analytics_request = loop { + let requests = server.received_requests().await.unwrap_or_default(); + if let Some(request) = requests + .into_iter() + .find(|request| request.url.path() == "/codex/analytics-events/events") + { + break request; + } + if Instant::now() >= deadline { + panic!("timed out waiting for turn analytics request"); + } + tokio::time::sleep(Duration::from_millis(50)).await; + }; + + let payload: Value = serde_json::from_slice(&analytics_request.body)?; + let event = payload["events"] + .as_array() + .and_then(|events| { + events + .iter() + .find(|event| event["event_type"] == "codex_turn_metadata") + }) + .expect("codex_turn_metadata event should be present"); + + let event_params = &event["event_params"]; + + assert_eq!(event_params["sandbox_policy"], "read_only"); + assert_eq!( + event_params["product_client_id"], + serde_json::json!(codex_core::default_client::originator().value) + ); + assert_eq!(event_params["model_slug"], session_configured.model); + assert_eq!(event_params["effort"], "high"); + assert_eq!(event_params["summary"], "detailed"); + assert_eq!(event_params["service_tier"], "flex"); + assert_eq!(event_params["collaboration_mode"], "plan"); + assert!(event_params["thread_id"].as_str().is_some()); + assert!(event_params["turn_id"].as_str().is_some()); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn assistant_message_item_is_emitted() -> anyhow::Result<()> { skip_if_no_network!(Ok(()));