From 1d220d93257f12cfbe08f5ae8595cce3b6fe3a1e Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 7 Apr 2026 12:20:29 -0700 Subject: [PATCH 1/5] [codex-analytics] feature plumbing and emittance --- .../analytics/src/analytics_client_tests.rs | 462 ++++++++++++++++++ codex-rs/analytics/src/client.rs | 22 + codex-rs/analytics/src/events.rs | 45 ++ codex-rs/analytics/src/facts.rs | 44 ++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 377 +++++++++++++- .../app-server/src/bespoke_event_handling.rs | 33 ++ .../app-server/src/codex_message_processor.rs | 12 + codex-rs/app-server/src/message_processor.rs | 20 +- codex-rs/app-server/tests/common/config.rs | 28 ++ codex-rs/app-server/tests/common/lib.rs | 1 + .../app-server/tests/suite/v2/analytics.rs | 49 ++ .../tests/suite/v2/turn_interrupt.rs | 21 +- .../app-server/tests/suite/v2/turn_start.rs | 142 ++++++ codex-rs/core/src/codex.rs | 65 ++- codex-rs/core/src/codex_delegate.rs | 1 + codex-rs/core/src/codex_tests.rs | 1 + codex-rs/core/src/codex_tests_guardian.rs | 1 + codex-rs/core/src/prompt_debug.rs | 1 + codex-rs/core/src/state/session.rs | 12 + codex-rs/core/src/thread_manager.rs | 6 + codex-rs/core/src/thread_manager_tests.rs | 4 + .../core/src/thread_rollout_truncation.rs | 12 + codex-rs/core/tests/common/test_codex.rs | 1 + codex-rs/core/tests/suite/client.rs | 1 + codex-rs/mcp-server/src/message_processor.rs | 1 + codex-rs/protocol/src/protocol.rs | 8 + 27 files changed, 1331 insertions(+), 42 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 73ea42d760..728a44ace0 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -6,6 +6,7 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::CodexTurnEventRequest; use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; @@ -27,28 +28,43 @@ use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer; use codex_app_server_protocol::AskForApproval as AppServerAskForApproval; use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy; +use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::SessionSource as AppServerSessionSource; use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus; +use codex_app_server_protocol::Turn; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnError as AppServerTurnError; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; +use codex_app_server_protocol::UserInput; use codex_login::default_client::DEFAULT_ORIGINATOR; use codex_login::default_client::originator; use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SubAgentSource; use pretty_assertions::assert_eq; use serde_json::json; @@ -114,6 +130,179 @@ fn sample_thread_resume_response(thread_id: &str, ephemeral: bool, model: &str) } } +fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest { + ClientRequest::TurnStart { + request_id: RequestId::Integer(request_id), + params: TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![ + UserInput::Text { + text: "hello".to_string(), + text_elements: vec![], + }, + UserInput::Image { + url: "https://example.com/a.png".to_string(), + }, + ], + ..Default::default() + }, + } +} + +fn sample_turn_start_response(turn_id: &str, request_id: i64) -> ClientResponse { + ClientResponse::TurnStart { + request_id: RequestId::Integer(request_id), + response: codex_app_server_protocol::TurnStartResponse { + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + started_at: None, + completed_at: None, + duration_ms: None, + }, + }, + } +} + +fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNotification { + ServerNotification::TurnStarted(TurnStartedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + started_at: Some(455), + completed_at: None, + duration_ms: None, + }, + }) +} + +fn sample_turn_completed_notification( + thread_id: &str, + turn_id: &str, + status: AppServerTurnStatus, + codex_error_info: Option, +) -> ServerNotification { + ServerNotification::TurnCompleted(TurnCompletedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status, + error: codex_error_info.map(|codex_error_info| AppServerTurnError { + message: "turn failed".to_string(), + codex_error_info: Some(codex_error_info), + additional_details: None, + }), + started_at: None, + completed_at: Some(456), + duration_ms: Some(1234), + }, + }) +} + +fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { + TurnResolvedConfigFact { + turn_id: turn_id.to_string(), + thread_id: "thread-2".to_string(), + num_input_images: 1, + submission_type: None, + model: "gpt-5".to_string(), + model_provider: "openai".to_string(), + sandbox_policy: SandboxPolicy::new_read_only_policy(), + reasoning_effort: None, + reasoning_summary: None, + service_tier: None, + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: ApprovalsReviewer::GuardianSubagent, + sandbox_network_access: true, + collaboration_mode: ModeKind::Plan, + personality: None, + is_first_turn: true, + } +} + +async fn ingest_turn_prerequisites( + reducer: &mut AnalyticsReducer, + out: &mut Vec, + include_initialize: bool, + include_resolved_config: bool, + include_started: bool, +) { + if include_initialize { + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: CodexRuntimeMetadata { + codex_rs_version: "0.1.0".to_string(), + runtime_os: "macos".to_string(), + runtime_os_version: "15.3.1".to_string(), + runtime_arch: "aarch64".to_string(), + }, + rpc_transport: AppServerRpcTransport::Stdio, + }, + out, + ) + .await; + } + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)), + }, + out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)), + }, + out, + ) + .await; + + if include_resolved_config { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + sample_turn_resolved_config("turn-2"), + ))), + out, + ) + .await; + } + + if include_started { + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-2", "turn-2", + ))), + out, + ) + .await; + } +} + fn expected_absolute_path(path: &PathBuf) -> String { std::fs::canonicalize(path) .unwrap_or_else(|_| path.to_path_buf()) @@ -823,6 +1012,279 @@ async fn reducer_ingests_plugin_state_changed_fact() { ); } +#[test] +fn turn_event_serializes_expected_shape() { + let event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: crate::events::CodexTurnEventParams { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + product_client_id: "codex-tui".to_string(), + submission_type: None, + model: Some("gpt-5".to_string()), + model_provider: "openai".to_string(), + sandbox_policy: Some("read_only"), + reasoning_effort: Some("high".to_string()), + reasoning_summary: Some("detailed".to_string()), + service_tier: "flex".to_string(), + approval_policy: "on-request".to_string(), + approvals_reviewer: "guardian_subagent".to_string(), + sandbox_network_access: true, + collaboration_mode: Some("plan"), + personality: Some("pragmatic".to_string()), + num_input_images: 2, + is_first_turn: true, + status: Some(TurnStatus::Completed), + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: Some(1234), + started_at: Some(455), + completed_at: Some(456), + }, + })); + + let payload = serde_json::to_value(&event).expect("serialize turn event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_turn_event", + "event_params": { + "thread_id": "thread-2", + "turn_id": "turn-2", + "product_client_id": "codex-tui", + "submission_type": null, + "model": "gpt-5", + "model_provider": "openai", + "sandbox_policy": "read_only", + "reasoning_effort": "high", + "reasoning_summary": "detailed", + "service_tier": "flex", + "approval_policy": "on-request", + "approvals_reviewer": "guardian_subagent", + "sandbox_network_access": true, + "collaboration_mode": "plan", + "personality": "pragmatic", + "num_input_images": 2, + "is_first_turn": true, + "status": "completed", + "turn_error": null, + "steer_count": null, + "total_tool_call_count": null, + "shell_command_count": null, + "file_change_count": null, + "mcp_tool_call_count": null, + "dynamic_tool_call_count": null, + "subagent_tool_call_count": null, + "web_search_count": null, + "image_generation_count": null, + "duration_ms": 1234, + "started_at": 455, + "completed_at": 456 + } + }) + ); +} + +#[tokio::test] +async fn turn_lifecycle_emits_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_type"], json!("codex_turn_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); + assert_eq!( + payload["event_params"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["started_at"], json!(455)); + assert_eq!(payload["event_params"]["completed_at"], json!(456)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + +#[tokio::test] +async fn turn_does_not_emit_without_required_prerequisites() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ false, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!( + payload["event_params"]["product_client_id"], + json!(originator().value) + ); + + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert!(out.is_empty()); +} + +#[tokio::test] +async fn turn_lifecycle_emits_failed_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Failed, + Some(codex_app_server_protocol::CodexErrorInfo::BadRequest), + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["status"], json!("failed")); + assert_eq!(payload["event_params"]["turn_error"], json!("badRequest")); +} + +#[tokio::test] +async fn turn_lifecycle_emits_interrupted_turn_event_without_error() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Interrupted, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["status"], json!("interrupted")); + assert_eq!(payload["event_params"]["turn_error"], json!(null)); +} + +#[tokio::test] +async fn turn_completed_without_started_notification_emits_null_started_at() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["started_at"], json!(null)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + fn sample_plugin_metadata() -> PluginTelemetryMetadata { PluginTelemetryMetadata { plugin_id: PluginId::parse("sample@test").expect("valid plugin id"), diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index dd300a7bd6..e61affe76c 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -13,9 +13,13 @@ use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; use crate::reducer::AnalyticsReducer; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; use codex_login::AuthManager; use codex_login::default_client::create_client; use codex_plugin::PluginTelemetryMetadata; @@ -160,6 +164,14 @@ impl AnalyticsEventsClient { ))); } + pub fn track_request(&self, connection_id: u64, request_id: RequestId, request: ClientRequest) { + self.record_fact(AnalyticsFact::Request { + connection_id, + request_id, + request: Box::new(request), + }); + } + pub fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) { if !self.queue.should_enqueue_app_used(&tracking, &app) { return; @@ -178,6 +190,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), + )); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { @@ -227,6 +245,10 @@ impl AnalyticsEventsClient { response: Box::new(response), }); } + + pub fn track_notification(&self, notification: ServerNotification) { + self.record_fact(AnalyticsFact::Notification(Box::new(notification))); + } } async fn send_track_events( diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 885e93bbb9..1947fd506d 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -3,6 +3,9 @@ use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnStatus; +use crate::facts::TurnSubmissionType; +use codex_app_server_protocol::CodexErrorInfo; use codex_login::default_client::originator; use codex_plugin::PluginTelemetryMetadata; use codex_protocol::protocol::SessionSource; @@ -37,6 +40,7 @@ pub(crate) enum TrackEventRequest { ThreadInitialized(ThreadInitializedEvent), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), + TurnEvent(Box), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -122,6 +126,47 @@ pub(crate) struct CodexAppUsedEventRequest { pub(crate) event_params: CodexAppMetadata, } +#[derive(Serialize)] +pub(crate) struct CodexTurnEventParams { + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) product_client_id: String, + pub(crate) submission_type: Option, + pub(crate) model: Option, + pub(crate) model_provider: String, + pub(crate) sandbox_policy: Option<&'static str>, + pub(crate) reasoning_effort: Option, + pub(crate) reasoning_summary: Option, + pub(crate) service_tier: String, + pub(crate) approval_policy: String, + pub(crate) approvals_reviewer: String, + pub(crate) sandbox_network_access: bool, + pub(crate) collaboration_mode: Option<&'static str>, + pub(crate) personality: Option, + pub(crate) num_input_images: usize, + pub(crate) is_first_turn: bool, + pub(crate) status: Option, + pub(crate) turn_error: Option, + pub(crate) steer_count: Option, + pub(crate) total_tool_call_count: Option, + pub(crate) shell_command_count: Option, + pub(crate) file_change_count: Option, + pub(crate) mcp_tool_call_count: Option, + pub(crate) dynamic_tool_call_count: Option, + pub(crate) subagent_tool_call_count: Option, + pub(crate) web_search_count: Option, + pub(crate) image_generation_count: Option, + pub(crate) duration_ms: Option, + pub(crate) started_at: Option, + pub(crate) completed_at: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexTurnEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexTurnEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexPluginMetadata { pub(crate) plugin_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index e19d15d847..fdc45cb9b3 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -6,6 +6,14 @@ use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; use serde::Serialize; @@ -30,6 +38,41 @@ pub fn build_track_events_context( } } +#[derive(Clone)] +pub struct TurnResolvedConfigFact { + pub turn_id: String, + pub thread_id: String, + pub num_input_images: usize, + pub submission_type: Option, + pub model: String, + pub model_provider: String, + pub sandbox_policy: SandboxPolicy, + pub reasoning_effort: Option, + pub reasoning_summary: Option, + pub service_tier: Option, + pub approval_policy: AskForApproval, + pub approvals_reviewer: ApprovalsReviewer, + pub sandbox_network_access: bool, + pub collaboration_mode: ModeKind, + pub personality: Option, + pub is_first_turn: bool, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, +} + #[derive(Clone, Debug)] pub struct SkillInvocation { pub skill_name: String, @@ -89,6 +132,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), + TurnResolvedConfig(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index f2f76ca8cf..d3193c6fd1 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -10,6 +10,9 @@ pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; pub use facts::TrackEventsContext; +pub use facts::TurnResolvedConfigFact; +pub use facts::TurnStatus; +pub use facts::TurnSubmissionType; pub use facts::build_track_events_context; #[cfg(test)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 63b9c3d5be..80e6786af5 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -5,6 +5,8 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::CodexTurnEventParams; +use crate::events::CodexTurnEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializationMode; @@ -26,11 +28,22 @@ use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::UserInput; use codex_git_utils::collect_git_info; use codex_git_utils::get_git_repo_root; use codex_login::default_client::originator; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use sha1::Digest; @@ -39,6 +52,8 @@ use std::path::Path; #[derive(Default)] pub(crate) struct AnalyticsReducer { + requests: HashMap<(u64, RequestId), RequestState>, + turns: HashMap, connections: HashMap, } @@ -47,6 +62,32 @@ struct ConnectionState { runtime: CodexRuntimeMetadata, } +enum RequestState { + TurnStart(PendingTurnStartState), +} + +struct PendingTurnStartState { + thread_id: String, + num_input_images: usize, +} + +#[derive(Clone)] +struct CompletedTurnState { + status: Option, + turn_error: Option, + completed_at: u64, + duration_ms: Option, +} + +struct TurnState { + connection_id: Option, + thread_id: Option, + num_input_images: Option, + resolved_config: Option, + started_at: Option, + completed: Option, +} + impl AnalyticsReducer { pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec) { match input { @@ -66,21 +107,28 @@ impl AnalyticsReducer { ); } AnalyticsFact::Request { - connection_id: _connection_id, - request_id: _request_id, - request: _request, - } => {} + connection_id, + request_id, + request, + } => { + self.ingest_request(connection_id, request_id, *request); + } AnalyticsFact::Response { connection_id, response, } => { self.ingest_response(connection_id, *response, out); } - AnalyticsFact::Notification(_notification) => {} + AnalyticsFact::Notification(notification) => { + self.ingest_notification(*notification, out); + } AnalyticsFact::Custom(input) => match input { CustomAnalyticsFact::SubAgentThreadStarted(input) => { self.ingest_subagent_thread_started(input, out); } + CustomAnalyticsFact::TurnResolvedConfig(input) => { + self.ingest_turn_resolved_config(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -135,6 +183,52 @@ impl AnalyticsReducer { )); } + fn ingest_request( + &mut self, + connection_id: u64, + request_id: RequestId, + request: ClientRequest, + ) { + let ClientRequest::TurnStart { params, .. } = request else { + return; + }; + self.requests.insert( + (connection_id, request_id), + RequestState::TurnStart(PendingTurnStartState { + thread_id: params.thread_id, + num_input_images: params + .input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + }), + ); + } + + fn ingest_turn_resolved_config( + &mut self, + input: TurnResolvedConfigFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let thread_id = input.thread_id.clone(); + let num_input_images = input.num_input_images; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.thread_id = Some(thread_id); + turn_state.num_input_images = Some(num_input_images); + turn_state.resolved_config = Some(input); + self.maybe_emit_turn_event(&turn_id, out); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -235,24 +329,124 @@ impl AnalyticsReducer { response: ClientResponse, out: &mut Vec, ) { - let (thread, model, initialization_mode) = match response { - ClientResponse::ThreadStart { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::New, - ), - ClientResponse::ThreadResume { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Resumed, - ), - ClientResponse::ThreadFork { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Forked, - ), - _ => return, - }; + match response { + ClientResponse::ThreadStart { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::New, + out, + ); + } + ClientResponse::ThreadResume { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Resumed, + out, + ); + } + ClientResponse::ThreadFork { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Forked, + out, + ); + } + ClientResponse::TurnStart { + request_id, + response, + } => { + let turn_id = response.turn.id; + let Some(RequestState::TurnStart(pending_request)) = + self.requests.remove(&(connection_id, request_id)) + else { + return; + }; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.connection_id = Some(connection_id); + turn_state.thread_id = Some(pending_request.thread_id); + turn_state.num_input_images = Some(pending_request.num_input_images); + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn ingest_notification( + &mut self, + notification: ServerNotification, + out: &mut Vec, + ) { + match notification { + ServerNotification::TurnStarted(notification) => { + let turn_state = self.turns.entry(notification.turn.id).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.started_at = notification + .turn + .started_at + .and_then(|started_at| u64::try_from(started_at).ok()); + } + ServerNotification::TurnCompleted(notification) => { + let turn_state = + self.turns + .entry(notification.turn.id.clone()) + .or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + completed: None, + }); + turn_state.completed = Some(CompletedTurnState { + status: analytics_turn_status(notification.turn.status), + turn_error: notification + .turn + .error + .and_then(|error| error.codex_error_info), + completed_at: notification + .turn + .completed_at + .and_then(|completed_at| u64::try_from(completed_at).ok()) + .unwrap_or_default(), + duration_ms: notification + .turn + .duration_ms + .and_then(|duration_ms| u64::try_from(duration_ms).ok()), + }); + let turn_id = notification.turn.id; + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn emit_thread_initialized( + &mut self, + connection_id: u64, + thread: codex_app_server_protocol::Thread, + model: String, + initialization_mode: ThreadInitializationMode, + out: &mut Vec, + ) { let thread_source: SessionSource = thread.source.into(); let Some(connection_state) = self.connections.get(&connection_id) else { return; @@ -275,6 +469,143 @@ impl AnalyticsReducer { }, )); } + + fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec) { + let Some(turn_state) = self.turns.get(turn_id) else { + return; + }; + if turn_state.thread_id.is_none() + || turn_state.num_input_images.is_none() + || turn_state.resolved_config.is_none() + || turn_state.completed.is_none() + { + return; + } + let product_client_id = turn_state + .connection_id + .and_then(|connection_id| self.connections.get(&connection_id)) + .map(|connection_state| connection_state.app_server_client.product_client_id.clone()) + .unwrap_or_else(|| originator().value); + out.push(TrackEventRequest::TurnEvent(Box::new( + CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: codex_turn_event_params( + product_client_id, + turn_id.to_string(), + turn_state, + ), + }, + ))); + self.turns.remove(turn_id); + } +} + +fn codex_turn_event_params( + product_client_id: String, + turn_id: String, + turn_state: &TurnState, +) -> CodexTurnEventParams { + let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = ( + turn_state.thread_id.clone(), + turn_state.num_input_images, + turn_state.resolved_config.clone(), + turn_state.completed.clone(), + ) else { + unreachable!("turn event params require a fully populated turn state"); + }; + let started_at = turn_state.started_at; + let TurnResolvedConfigFact { + turn_id: _resolved_turn_id, + thread_id: _resolved_thread_id, + num_input_images: _resolved_num_input_images, + submission_type, + model, + model_provider, + sandbox_policy, + reasoning_effort, + reasoning_summary, + service_tier, + approval_policy, + approvals_reviewer, + sandbox_network_access, + collaboration_mode, + personality, + is_first_turn, + } = resolved_config; + CodexTurnEventParams { + thread_id, + turn_id, + product_client_id, + submission_type, + model: Some(model), + model_provider, + sandbox_policy: Some(sandbox_policy_mode(&sandbox_policy)), + reasoning_effort: reasoning_effort.map(|value| value.to_string()), + reasoning_summary: reasoning_summary_mode(reasoning_summary), + service_tier: service_tier + .map(|value| value.to_string()) + .unwrap_or_else(|| "default".to_string()), + approval_policy: approval_policy.to_string(), + approvals_reviewer: approvals_reviewer.to_string(), + sandbox_network_access, + collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), + personality: personality_mode(personality), + num_input_images, + is_first_turn, + status: completed.status, + turn_error: completed.turn_error, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: completed.duration_ms, + started_at, + completed_at: Some(completed.completed_at), + } +} + +fn sandbox_policy_mode(sandbox_policy: &SandboxPolicy) -> &'static str { + match sandbox_policy { + SandboxPolicy::DangerFullAccess => "full_access", + SandboxPolicy::ReadOnly { .. } => "read_only", + SandboxPolicy::WorkspaceWrite { .. } => "workspace_write", + SandboxPolicy::ExternalSandbox { .. } => "external_sandbox", + } +} + +fn collaboration_mode_mode(mode: ModeKind) -> &'static str { + match mode { + ModeKind::Plan => "plan", + ModeKind::Default | ModeKind::PairProgramming | ModeKind::Execute => "default", + } +} + +fn reasoning_summary_mode(summary: Option) -> Option { + match summary { + Some(ReasoningSummary::None) | None => None, + Some(summary) => Some(summary.to_string()), + } +} + +fn personality_mode(personality: Option) -> Option { + match personality { + Some(Personality::None) | None => None, + Some(personality) => Some(personality.to_string()), + } +} + +fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Option { + match status { + codex_app_server_protocol::TurnStatus::Completed => Some(TurnStatus::Completed), + codex_app_server_protocol::TurnStatus::Failed => Some(TurnStatus::Failed), + codex_app_server_protocol::TurnStatus::Interrupted => Some(TurnStatus::Interrupted), + codex_app_server_protocol::TurnStatus::InProgress => None, + } } pub(crate) fn skill_id_for_local_skill( diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 141ee78bdb..ec42f0b611 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -12,6 +12,7 @@ use crate::thread_state::TurnSummary; use crate::thread_state::resolve_server_request_on_thread_listener; use crate::thread_status::ThreadWatchActiveGuard; use crate::thread_status::ThreadWatchManager; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::AccountRateLimitsUpdatedNotification; use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermissionProfile; use codex_app_server_protocol::AgentMessageDeltaNotification; @@ -166,6 +167,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id: ThreadId, conversation: Arc, thread_manager: Arc, + analytics_events_client: Option, outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, thread_watch_manager: ThreadWatchManager, @@ -201,6 +203,10 @@ pub(crate) async fn apply_bespoke_event_handling( thread_id: conversation_id.to_string(), turn, }; + if let Some(analytics_events_client) = analytics_events_client.as_ref() { + analytics_events_client + .track_notification(ServerNotification::TurnStarted(notification.clone())); + } outgoing .send_server_notification(ServerNotification::TurnStarted(notification)) .await; @@ -217,6 +223,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id, event_turn_id, turn_complete_event, + analytics_events_client.as_ref(), &outgoing, &thread_state, ) @@ -1720,6 +1727,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id, event_turn_id, turn_aborted_event, + analytics_events_client.as_ref(), &outgoing, &thread_state, ) @@ -1897,6 +1905,7 @@ async fn emit_turn_completed_with_status( conversation_id: ThreadId, event_turn_id: String, turn_completion_metadata: TurnCompletionMetadata, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, ) { let notification = TurnCompletedNotification { @@ -1911,6 +1920,10 @@ async fn emit_turn_completed_with_status( duration_ms: turn_completion_metadata.duration_ms, }, }; + if let Some(analytics_events_client) = analytics_events_client { + analytics_events_client + .track_notification(ServerNotification::TurnCompleted(notification.clone())); + } outgoing .send_server_notification(ServerNotification::TurnCompleted(notification)) .await; @@ -2103,6 +2116,7 @@ async fn handle_turn_complete( conversation_id: ThreadId, event_turn_id: String, turn_complete_event: TurnCompleteEvent, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { @@ -2123,6 +2137,7 @@ async fn handle_turn_complete( completed_at: turn_complete_event.completed_at, duration_ms: turn_complete_event.duration_ms, }, + analytics_events_client, outgoing, ) .await; @@ -2132,6 +2147,7 @@ async fn handle_turn_interrupted( conversation_id: ThreadId, event_turn_id: String, turn_aborted_event: TurnAbortedEvent, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { @@ -2147,6 +2163,7 @@ async fn handle_turn_interrupted( completed_at: turn_aborted_event.completed_at, duration_ms: turn_aborted_event.duration_ms, }, + analytics_events_client, outgoing, ) .await; @@ -2885,6 +2902,7 @@ mod tests { use codex_app_server_protocol::GuardianApprovalReviewStatus; use codex_app_server_protocol::JSONRPCErrorError; use codex_app_server_protocol::TurnPlanStepStatus; + use codex_login::AuthManager; use codex_login::CodexAuth; use codex_protocol::items::HookPromptFragment; use codex_protocol::items::build_hook_prompt_message; @@ -3006,6 +3024,7 @@ mod tests { outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, thread_watch_manager: ThreadWatchManager, + analytics_events_client: AnalyticsEventsClient, codex_home: PathBuf, } @@ -3020,6 +3039,7 @@ mod tests { self.conversation_id, self.conversation.clone(), self.thread_manager.clone(), + Some(self.analytics_events_client.clone()), self.outgoing.clone(), self.thread_state.clone(), self.thread_watch_manager.clone(), @@ -3328,6 +3348,13 @@ mod tests { outgoing: outgoing.clone(), thread_state: thread_state.clone(), thread_watch_manager: thread_watch_manager.clone(), + analytics_events_client: AnalyticsEventsClient::new( + AuthManager::from_auth_for_testing( + CodexAuth::create_dummy_chatgpt_auth_for_testing(), + ), + "http://localhost".to_string(), + Some(false), + ), codex_home: codex_home.path().to_path_buf(), }; @@ -3736,6 +3763,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_complete_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -3784,6 +3812,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_aborted_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -3831,6 +3860,7 @@ mod tests { conversation_id, event_turn_id.clone(), turn_complete_event(&event_turn_id), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4097,6 +4127,7 @@ mod tests { conversation_a, a_turn1.clone(), turn_complete_event(&a_turn1), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4118,6 +4149,7 @@ mod tests { conversation_b, b_turn1.clone(), turn_complete_event(&b_turn1), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -4129,6 +4161,7 @@ mod tests { conversation_a, a_turn2.clone(), turn_complete_event(&a_turn2), + /*analytics_events_client*/ None, &outgoing, &thread_state, ) diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 91641898d1..4c19fba551 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6670,6 +6670,15 @@ impl CodexMessageProcessor { }; let response = TurnStartResponse { turn }; + if self.config.features.enabled(Feature::GeneralAnalytics) { + self.analytics_events_client.track_response( + request_id.connection_id.0, + ClientResponse::TurnStart { + request_id: request_id.request_id.clone(), + response: response.clone(), + }, + ); + } self.outgoing.send_response(request_id, response).await; } Err(err) => { @@ -7446,6 +7455,9 @@ impl CodexMessageProcessor { conversation_id, conversation.clone(), thread_manager.clone(), + listener_task_context + .general_analytics_enabled + .then(|| listener_task_context.analytics_events_client.clone()), thread_outgoing, thread_state.clone(), thread_watch_manager.clone(), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index 15df5a2a54..03c5a1ce96 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -219,6 +219,11 @@ impl MessageProcessor { auth_manager.set_external_auth(Arc::new(ExternalAuthRefreshBridge { outgoing: outgoing.clone(), })); + let analytics_events_client = AnalyticsEventsClient::new( + Arc::clone(&auth_manager), + config.chatgpt_base_url.trim_end_matches('/').to_string(), + config.analytics_enabled, + ); let thread_manager = Arc::new(ThreadManager::new( config.as_ref(), auth_manager.clone(), @@ -229,12 +234,8 @@ impl MessageProcessor { .enabled(Feature::DefaultModeRequestUserInput), }, environment_manager, + Some(analytics_events_client.clone()), )); - let analytics_events_client = AnalyticsEventsClient::new( - Arc::clone(&auth_manager), - config.chatgpt_base_url.trim_end_matches('/').to_string(), - config.analytics_enabled, - ); thread_manager .plugins_manager() .set_analytics_events_client(analytics_events_client.clone()); @@ -671,6 +672,15 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + if self.config.features.enabled(Feature::GeneralAnalytics) + && let ClientRequest::TurnStart { request_id, .. } = &codex_request + { + self.analytics_events_client.track_request( + connection_id.0, + request_id.clone(), + codex_request.clone(), + ); + } match codex_request { ClientRequest::ConfigRead { request_id, params } => { diff --git a/codex-rs/app-server/tests/common/config.rs b/codex-rs/app-server/tests/common/config.rs index deb16c6322..1ac2572fa2 100644 --- a/codex-rs/app-server/tests/common/config.rs +++ b/codex-rs/app-server/tests/common/config.rs @@ -78,3 +78,31 @@ model_provider = "{model_provider_id}" ), ) } + +pub fn write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home: &Path, + server_uri: &str, + chatgpt_base_url: &str, +) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +chatgpt_base_url = "{chatgpt_base_url}" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 3f89765851..90553760d9 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -14,6 +14,7 @@ pub use auth_fixtures::encode_id_token; pub use auth_fixtures::write_chatgpt_auth; use codex_app_server_protocol::JSONRPCResponse; pub use config::write_mock_responses_config_toml; +pub use config::write_mock_responses_config_toml_with_chatgpt_base_url; pub use core_test_support::format_with_current_shell; pub use core_test_support::format_with_current_shell_display; pub use core_test_support::format_with_current_shell_display_non_login; diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index a4d7a7f349..babf6396d8 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -80,6 +80,20 @@ async fn app_server_default_analytics_enabled_with_flag() -> Result<()> { } pub(crate) async fn enable_analytics_capture(server: &MockServer, codex_home: &Path) -> Result<()> { + let config_path = codex_home.join("config.toml"); + let config_toml = std::fs::read_to_string(&config_path)?; + if !config_toml.contains("[features]") { + std::fs::write( + &config_path, + format!("{config_toml}\n[features]\ngeneral_analytics = true\n"), + )?; + } else if !config_toml.contains("general_analytics") { + std::fs::write( + &config_path, + config_toml.replace("[features]\n", "[features]\ngeneral_analytics = true\n"), + )?; + } + Mock::given(method("POST")) .and(path("/codex/analytics-events/events")) .respond_with(ResponseTemplate::new(200)) @@ -120,6 +134,41 @@ pub(crate) async fn wait_for_analytics_payload( serde_json::from_slice(&body).map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}")) } +pub(crate) async fn wait_for_analytics_event( + server: &MockServer, + read_timeout: Duration, + event_type: &str, +) -> Result { + timeout(read_timeout, async { + loop { + let Some(requests) = server.received_requests().await else { + tokio::time::sleep(Duration::from_millis(25)).await; + continue; + }; + for request in &requests { + if request.method != "POST" + || request.url.path() != "/codex/analytics-events/events" + { + continue; + } + let payload: Value = serde_json::from_slice(&request.body) + .map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?; + let Some(events) = payload["events"].as_array() else { + continue; + }; + if let Some(event) = events + .iter() + .find(|event| event["event_type"] == event_type) + { + return Ok::(event.clone()); + } + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await? +} + pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> { let events = payload["events"] .as_array() diff --git a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs index 2850c7b74f..b553137752 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_interrupt.rs @@ -3,6 +3,7 @@ use anyhow::Result; use app_test_support::McpProcess; use app_test_support::create_mock_responses_server_sequence; +use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; use codex_app_server_protocol::JSONRPCNotification; @@ -43,14 +44,15 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { std::fs::create_dir(&working_directory)?; // Mock server: long-running shell command then (after abort) nothing else needed. - let server = create_mock_responses_server_sequence(vec![create_shell_command_sse_response( - shell_command.clone(), - Some(&working_directory), - Some(10_000), - "call_sleep", - )?]) - .await; - create_config_toml(&codex_home, &server.uri(), "never", "danger-full-access")?; + let server = + create_mock_responses_server_sequence_unchecked(vec![create_shell_command_sse_response( + shell_command.clone(), + Some(&working_directory), + Some(10_000), + "call_sleep", + )?]) + .await; + create_config_toml(&codex_home, &server.uri(), "never", "workspace-write")?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -87,6 +89,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { ) .await??; let TurnStartResponse { turn } = to_response::(turn_resp)?; + let turn_id = turn.id.clone(); // Give the command a brief moment to start. tokio::time::sleep(std::time::Duration::from_secs(1)).await; @@ -96,7 +99,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> { let interrupt_id = mcp .send_turn_interrupt_request(TurnInterruptParams { thread_id: thread_id.clone(), - turn_id: turn.id, + turn_id: turn_id.clone(), }) .await?; let interrupt_resp: JSONRPCResponse = timeout( diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index e8f3e1df2c..5645a4d3a8 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use app_test_support::DEFAULT_CLIENT_NAME; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; use app_test_support::create_exec_command_sse_response; @@ -9,6 +10,7 @@ use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::format_with_current_shell_display; use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::ByteRange; @@ -64,6 +66,10 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +use super::analytics::enable_analytics_capture; +use super::analytics::wait_for_analytics_event; +use super::analytics::wait_for_analytics_payload; + #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); #[cfg(not(windows))] @@ -335,6 +341,142 @@ async fn turn_start_emits_user_message_item_with_text_elements() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_tracks_turn_event_analytics() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Image { + url: "https://example.com/a.png".to_string(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["turn_id"], turn.id); + assert_eq!( + event["event_params"]["product_client_id"], + DEFAULT_CLIENT_NAME + ); + assert_eq!(event["event_params"]["model"], "mock-model"); + assert_eq!(event["event_params"]["model_provider"], "mock_provider"); + assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); + assert_eq!(event["event_params"]["num_input_images"], 1); + assert_eq!(event["event_params"]["status"], "completed"); + assert!(event["event_params"]["started_at"].as_u64().is_some()); + assert!(event["event_params"]["completed_at"].as_u64().is_some()); + assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + + Ok(()) +} + +#[tokio::test] +async fn turn_start_does_not_track_turn_event_analytics_without_feature() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, codex_home.path()).await?; + let config_path = codex_home.path().join("config.toml"); + let config_toml = std::fs::read_to_string(&config_path)?; + std::fs::write( + &config_path, + config_toml.replace("general_analytics = true", "general_analytics = false"), + )?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "hello".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let _ = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let payload = wait_for_analytics_payload(&server, std::time::Duration::from_millis(250)).await; + assert!( + payload.is_err(), + "turn analytics should be gated off when general_analytics is disabled" + ); + Ok(()) +} + #[tokio::test] async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { let responses = vec![create_final_assistant_message_sse_response("Done")?]; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 237b77e38d..d8ab2e055c 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -51,6 +51,7 @@ use codex_analytics::AnalyticsEventsClient; use codex_analytics::AppInvocation; use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; +use codex_analytics::TurnResolvedConfigFact; use codex_analytics::build_track_events_context; use codex_app_server_protocol::McpServerElicitationRequest; use codex_app_server_protocol::McpServerElicitationRequestParams; @@ -186,6 +187,7 @@ use crate::config::resolve_web_search_mode_for_turn; use crate::context_manager::ContextManager; use crate::context_manager::TotalTokenUsageBreakdown; use crate::environment_context::EnvironmentContext; +use crate::thread_rollout_truncation::initial_history_has_prior_user_turns; use codex_config::CONFIG_TOML_FILE; use codex_config::types::McpServerConfig; use codex_config::types::ShellEnvironmentPolicy; @@ -430,6 +432,7 @@ pub(crate) struct CodexSpawnArgs { pub(crate) inherited_exec_policy: Option>, pub(crate) user_shell_override: Option, pub(crate) parent_trace: Option, + pub(crate) analytics_events_client: Option, } pub(crate) const INITIAL_SUBMIT_ID: &str = ""; @@ -484,6 +487,7 @@ impl Codex { user_shell_override, inherited_exec_policy, parent_trace: _, + analytics_events_client, } = args; let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY); let (tx_event, rx_event) = async_channel::unbounded(); @@ -678,6 +682,7 @@ impl Codex { skills_watcher, agent_control, environment, + analytics_events_client, ) .await .map_err(|e| { @@ -1532,6 +1537,7 @@ impl Session { skills_watcher: Arc, agent_control: AgentControl, environment: Option>, + analytics_events_client: Option, ) -> anyhow::Result> { debug!( "Configuring session: model={}; provider={:?}", @@ -1936,11 +1942,13 @@ impl Session { ), shell_zsh_path: config.zsh_path.clone(), main_execve_wrapper_exe: config.main_execve_wrapper_exe.clone(), - analytics_events_client: AnalyticsEventsClient::new( - Arc::clone(&auth_manager), - config.chatgpt_base_url.trim_end_matches('/').to_string(), - config.analytics_enabled, - ), + analytics_events_client: analytics_events_client.unwrap_or_else(|| { + AnalyticsEventsClient::new( + Arc::clone(&auth_manager), + config.chatgpt_base_url.trim_end_matches('/').to_string(), + config.analytics_enabled, + ) + }), hooks, rollout: Mutex::new(rollout_recorder), user_shell: Arc::new(default_shell), @@ -2257,6 +2265,11 @@ impl Session { SessionSource::SubAgent(_) ) }; + let has_prior_user_turns = initial_history_has_prior_user_turns(&conversation_history); + { + let mut state = self.state.lock().await; + state.set_next_turn_is_first(!has_prior_user_turns); + } match conversation_history { InitialHistory::New => { // Defer initial context insertion until the first real turn starts so @@ -6035,6 +6048,8 @@ pub(crate) async fn run_turn( .await; } + track_turn_resolved_config_analytics(&sess, &turn_context, &input).await; + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) .await; @@ -6321,6 +6336,46 @@ pub(crate) async fn run_turn( last_agent_message } +async fn track_turn_resolved_config_analytics( + sess: &Session, + turn_context: &TurnContext, + input: &[UserInput], +) { + if !sess.enabled(Feature::GeneralAnalytics) { + return; + } + + let is_first_turn = { + let mut state = sess.state.lock().await; + state.take_next_turn_is_first() + }; + sess.services + .analytics_events_client + .track_turn_resolved_config(TurnResolvedConfigFact { + turn_id: turn_context.sub_id.clone(), + thread_id: sess.conversation_id.to_string(), + num_input_images: input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + submission_type: None, + model: turn_context.model_info.slug.clone(), + model_provider: turn_context.config.model_provider_id.clone(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + reasoning_effort: turn_context.reasoning_effort, + reasoning_summary: Some(turn_context.reasoning_summary), + service_tier: turn_context.config.service_tier, + approval_policy: turn_context.approval_policy.value(), + approvals_reviewer: turn_context.config.approvals_reviewer, + sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), + collaboration_mode: turn_context.collaboration_mode.mode, + personality: turn_context.personality, + is_first_turn, + }); +} + async fn run_pre_sampling_compact( sess: &Arc, turn_context: &Arc, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index ab6324efa2..aba82d2d54 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -95,6 +95,7 @@ pub(crate) async fn run_codex_thread_interactive( user_shell_override: None, inherited_exec_policy: Some(Arc::clone(&parent_session.services.exec_policy)), parent_trace: None, + analytics_events_client: Some(parent_session.services.analytics_events_client.clone()), }) .await?; if parent_session.enabled(codex_features::Feature::GeneralAnalytics) { diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index b0437edbfe..ae61cc0777 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -2626,6 +2626,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_zsh_path() { .await .expect("create environment"), )), + /*analytics_events_client*/ None, ) .await; diff --git a/codex-rs/core/src/codex_tests_guardian.rs b/codex-rs/core/src/codex_tests_guardian.rs index 4f60c2f28e..7738cbbe91 100644 --- a/codex-rs/core/src/codex_tests_guardian.rs +++ b/codex-rs/core/src/codex_tests_guardian.rs @@ -457,6 +457,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { inherited_exec_policy: Some(Arc::new(parent_exec_policy)), user_shell_override: None, parent_trace: None, + analytics_events_client: None, }) .await .expect("spawn guardian subagent"); diff --git a/codex-rs/core/src/prompt_debug.rs b/codex-rs/core/src/prompt_debug.rs index 5fe39ee88f..789aac5806 100644 --- a/codex-rs/core/src/prompt_debug.rs +++ b/codex-rs/core/src/prompt_debug.rs @@ -39,6 +39,7 @@ pub async fn build_prompt_input( .enabled(Feature::DefaultModeRequestUserInput), }, Arc::new(EnvironmentManager::from_env()), + /*analytics_events_client*/ None, ); let thread = thread_manager.start_thread(config).await?; diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 206f75060c..4360b16de4 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -33,6 +33,7 @@ pub(crate) struct SessionState { pub(crate) active_connector_selection: HashSet, pub(crate) pending_session_start_source: Option, granted_permissions: Option, + next_turn_is_first: bool, } impl SessionState { @@ -51,6 +52,7 @@ impl SessionState { active_connector_selection: HashSet::new(), pending_session_start_source: None, granted_permissions: None, + next_turn_is_first: true, } } @@ -73,6 +75,16 @@ impl SessionState { self.previous_turn_settings = previous_turn_settings; } + pub(crate) fn set_next_turn_is_first(&mut self, value: bool) { + self.next_turn_is_first = value; + } + + pub(crate) fn take_next_turn_is_first(&mut self) -> bool { + let is_first_turn = self.next_turn_is_first; + self.next_turn_is_first = false; + is_first_turn + } + pub(crate) fn clone_history(&self) -> ContextManager { self.history.clone() } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 6d93a1a329..e4e7a9b0ec 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -15,6 +15,7 @@ use crate::shell_snapshot::ShellSnapshot; use crate::skills_watcher::SkillsWatcher; use crate::skills_watcher::SkillsWatcherEvent; use crate::tasks::interrupted_turn_history_marker; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::TurnStatus; use codex_exec_server::EnvironmentManager; @@ -208,6 +209,7 @@ pub(crate) struct ThreadManagerState { mcp_manager: Arc, skills_watcher: Arc, session_source: SessionSource, + analytics_events_client: Option, // Captures submitted ops for testing purpose when test mode is enabled. ops_log: Option, } @@ -219,6 +221,7 @@ impl ThreadManager { session_source: SessionSource, collaboration_modes_config: CollaborationModesConfig, environment_manager: Arc, + analytics_events_client: Option, ) -> Self { let codex_home = config.codex_home.clone(); let restriction_product = session_source.restriction_product(); @@ -257,6 +260,7 @@ impl ThreadManager { skills_watcher, auth_manager, session_source, + analytics_events_client, ops_log: should_use_test_thread_manager_behavior() .then(|| Arc::new(std::sync::Mutex::new(Vec::new()))), }), @@ -326,6 +330,7 @@ impl ThreadManager { skills_watcher, auth_manager, session_source: SessionSource::Exec, + analytics_events_client: None, ops_log: should_use_test_thread_manager_behavior() .then(|| Arc::new(std::sync::Mutex::new(Vec::new()))), }), @@ -867,6 +872,7 @@ impl ThreadManagerState { inherited_exec_policy, user_shell_override, parent_trace, + analytics_events_client: self.analytics_events_client.clone(), }) .await?; self.finalize_thread_spawn(codex, thread_id, watch_registration) diff --git a/codex-rs/core/src/thread_manager_tests.rs b/codex-rs/core/src/thread_manager_tests.rs index 3db1735822..d3a56ce5ea 100644 --- a/codex-rs/core/src/thread_manager_tests.rs +++ b/codex-rs/core/src/thread_manager_tests.rs @@ -299,6 +299,7 @@ async fn new_uses_configured_openai_provider_for_model_refresh() { Arc::new(codex_exec_server::EnvironmentManager::new( /*exec_server_url*/ None, )), + /*analytics_events_client*/ None, ); let _ = manager.list_models(RefreshStrategy::Online).await; @@ -435,6 +436,7 @@ async fn interrupted_fork_snapshot_does_not_synthesize_turn_id_for_legacy_histor Arc::new(codex_exec_server::EnvironmentManager::new( /*exec_server_url*/ None, )), + /*analytics_events_client*/ None, ); let source = manager @@ -537,6 +539,7 @@ async fn interrupted_fork_snapshot_preserves_explicit_turn_id() { Arc::new(codex_exec_server::EnvironmentManager::new( /*exec_server_url*/ None, )), + /*analytics_events_client*/ None, ); let source = manager @@ -629,6 +632,7 @@ async fn interrupted_fork_snapshot_uses_persisted_mid_turn_history_without_live_ Arc::new(codex_exec_server::EnvironmentManager::new( /*exec_server_url*/ None, )), + /*analytics_events_client*/ None, ); let source = manager diff --git a/codex-rs/core/src/thread_rollout_truncation.rs b/codex-rs/core/src/thread_rollout_truncation.rs index 97370ce41e..e20ee53d47 100644 --- a/codex-rs/core/src/thread_rollout_truncation.rs +++ b/codex-rs/core/src/thread_rollout_truncation.rs @@ -8,9 +8,21 @@ use crate::event_mapping; use codex_protocol::items::TurnItem; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::EventMsg; +use codex_protocol::protocol::InitialHistory; use codex_protocol::protocol::InterAgentCommunication; use codex_protocol::protocol::RolloutItem; +pub(crate) fn initial_history_has_prior_user_turns(conversation_history: &InitialHistory) -> bool { + conversation_history.scan_rollout_items(rollout_item_is_user_turn_boundary) +} + +fn rollout_item_is_user_turn_boundary(item: &RolloutItem) -> bool { + match item { + RolloutItem::ResponseItem(item) => is_user_turn_boundary(item), + _ => false, + } +} + /// Return the indices of user message boundaries in a rollout. /// /// A user message boundary is a `RolloutItem::ResponseItem(ResponseItem::Message { .. })` diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index bffd0cbfd6..a6f3f0c72b 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -517,6 +517,7 @@ impl TestCodexBuilder { SessionSource::Exec, CollaborationModesConfig::default(), Arc::clone(&environment_manager), + /*analytics_events_client*/ None, ) } else { codex_core::test_support::thread_manager_with_models_provider_and_home( diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index cb97e4a3b3..51ade5f464 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -1122,6 +1122,7 @@ async fn prefers_apikey_when_config_prefers_apikey_even_with_chatgpt_tokens() { Arc::new(codex_exec_server::EnvironmentManager::new( /*exec_server_url*/ None, )), + /*analytics_events_client*/ None, ); let NewThread { thread: codex, .. } = thread_manager .start_thread(config) diff --git a/codex-rs/mcp-server/src/message_processor.rs b/codex-rs/mcp-server/src/message_processor.rs index 46470f994a..e0a1e77f68 100644 --- a/codex-rs/mcp-server/src/message_processor.rs +++ b/codex-rs/mcp-server/src/message_processor.rs @@ -70,6 +70,7 @@ impl MessageProcessor { .enabled(Feature::DefaultModeRequestUserInput), }, environment_manager, + /*analytics_events_client*/ None, )); Self { outgoing, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 05926a9544..0f4cd84e2f 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2277,6 +2277,14 @@ pub enum InitialHistory { } impl InitialHistory { + pub fn scan_rollout_items(&self, mut predicate: impl FnMut(&RolloutItem) -> bool) -> bool { + match self { + InitialHistory::New => false, + InitialHistory::Resumed(resumed) => resumed.history.iter().any(&mut predicate), + InitialHistory::Forked(items) => items.iter().any(predicate), + } + } + pub fn forked_from_id(&self) -> Option { match self { InitialHistory::New => None, From d3f463dce08f5c82cb29e8cf343e82d1bc1d2d08 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 7 Apr 2026 12:20:29 -0700 Subject: [PATCH 2/5] [codex-analytics] add token usage metadata --- .../analytics/src/analytics_client_tests.rs | 58 +++++++++++++++++++ codex-rs/analytics/src/client.rs | 7 +++ codex-rs/analytics/src/events.rs | 5 ++ codex-rs/analytics/src/facts.rs | 9 +++ codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 46 +++++++++++++++ .../app-server/src/bespoke_event_handling.rs | 18 +++++- .../app-server/tests/suite/v2/turn_start.rs | 5 ++ codex-rs/core/src/tasks/mod.rs | 8 +++ 9 files changed, 155 insertions(+), 2 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 728a44ace0..af7d957ea9 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -30,6 +30,7 @@ use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; @@ -66,6 +67,7 @@ use codex_protocol::config_types::ModeKind; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::TokenUsage; use pretty_assertions::assert_eq; use serde_json::json; use std::collections::HashSet; @@ -181,6 +183,20 @@ fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNot }) } +fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsageFact { + TurnTokenUsageFact { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_string(), + token_usage: TokenUsage { + total_tokens: 321, + input_tokens: 123, + cached_input_tokens: 45, + output_tokens: 140, + reasoning_output_tokens: 13, + }, + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -232,6 +248,7 @@ async fn ingest_turn_prerequisites( include_initialize: bool, include_resolved_config: bool, include_started: bool, + include_token_usage: bool, ) { if include_initialize { reducer @@ -301,6 +318,17 @@ async fn ingest_turn_prerequisites( ) .await; } + + if include_token_usage { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage(Box::new( + sample_turn_token_usage_fact("thread-2", "turn-2"), + ))), + out, + ) + .await; + } } fn expected_absolute_path(path: &PathBuf) -> String { @@ -1045,6 +1073,11 @@ fn turn_event_serializes_expected_shape() { subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, duration_ms: Some(1234), started_at: Some(455), completed_at: Some(456), @@ -1086,6 +1119,11 @@ fn turn_event_serializes_expected_shape() { "subagent_tool_call_count": null, "web_search_count": null, "image_generation_count": null, + "input_tokens": null, + "cached_input_tokens": null, + "output_tokens": null, + "reasoning_output_tokens": null, + "total_tokens": null, "duration_ms": 1234, "started_at": 455, "completed_at": 456 @@ -1105,6 +1143,7 @@ async fn turn_lifecycle_emits_turn_event() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ true, + /*include_token_usage*/ true, ) .await; reducer @@ -1133,6 +1172,14 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["started_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(123)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45)); + assert_eq!(payload["event_params"]["output_tokens"], json!(140)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(13) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } #[tokio::test] @@ -1146,6 +1193,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ false, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1175,6 +1223,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ true, /*include_resolved_config*/ false, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1266,6 +1315,7 @@ async fn turn_completed_without_started_notification_emits_null_started_at() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1283,6 +1333,14 @@ async fn turn_completed_without_started_notification_emits_null_started_at() { let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); assert_eq!(payload["event_params"]["started_at"], json!(null)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["output_tokens"], json!(null)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(null) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(null)); } fn sample_plugin_metadata() -> PluginTelemetryMetadata { diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index e61affe76c..41802bdb0e 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -14,6 +14,7 @@ use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; @@ -196,6 +197,12 @@ impl AnalyticsEventsClient { )); } + pub fn track_turn_token_usage(&self, fact: TurnTokenUsageFact) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnTokenUsage( + Box::new(fact), + ))); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 1947fd506d..48914d9de9 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -156,6 +156,11 @@ pub(crate) struct CodexTurnEventParams { pub(crate) subagent_tool_call_count: Option, pub(crate) web_search_count: Option, pub(crate) image_generation_count: Option, + pub(crate) input_tokens: Option, + pub(crate) cached_input_tokens: Option, + pub(crate) output_tokens: Option, + pub(crate) reasoning_output_tokens: Option, + pub(crate) total_tokens: Option, pub(crate) duration_ms: Option, pub(crate) started_at: Option, pub(crate) completed_at: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index fdc45cb9b3..ed5d96bb25 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -16,6 +16,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::TokenUsage; use serde::Serialize; use std::path::PathBuf; @@ -58,6 +59,13 @@ pub struct TurnResolvedConfigFact { pub is_first_turn: bool, } +#[derive(Clone)] +pub struct TurnTokenUsageFact { + pub turn_id: String, + pub thread_id: String, + pub token_usage: TokenUsage, +} + #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] pub enum TurnSubmissionType { @@ -133,6 +141,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), TurnResolvedConfig(Box), + TurnTokenUsage(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index d3193c6fd1..5a4630ff47 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -13,6 +13,7 @@ pub use facts::TrackEventsContext; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; pub use facts::TurnSubmissionType; +pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; #[cfg(test)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 80e6786af5..e24900ef04 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -30,6 +30,7 @@ use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnTokenUsageFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::CodexErrorInfo; @@ -46,6 +47,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; +use codex_protocol::protocol::TokenUsage; use sha1::Digest; use std::collections::HashMap; use std::path::Path; @@ -85,6 +87,7 @@ struct TurnState { num_input_images: Option, resolved_config: Option, started_at: Option, + token_usage: Option, completed: Option, } @@ -129,6 +132,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnResolvedConfig(input) => { self.ingest_turn_resolved_config(*input, out); } + CustomAnalyticsFact::TurnTokenUsage(input) => { + self.ingest_turn_token_usage(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -221,6 +227,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.thread_id = Some(thread_id); @@ -229,6 +236,26 @@ impl AnalyticsReducer { self.maybe_emit_turn_event(&turn_id, out); } + fn ingest_turn_token_usage( + &mut self, + input: TurnTokenUsageFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + token_usage: None, + completed: None, + }); + turn_state.thread_id = Some(input.thread_id); + turn_state.token_usage = Some(input.token_usage); + self.maybe_emit_turn_event(&turn_id, out); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -373,6 +400,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.connection_id = Some(connection_id); @@ -397,6 +425,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.started_at = notification @@ -414,6 +443,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, started_at: None, + token_usage: None, completed: None, }); turn_state.completed = Some(CompletedTurnState { @@ -532,6 +562,7 @@ fn codex_turn_event_params( personality, is_first_turn, } = resolved_config; + let token_usage = turn_state.token_usage.clone(); CodexTurnEventParams { thread_id, turn_id, @@ -563,6 +594,21 @@ fn codex_turn_event_params( subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.input_tokens), + cached_input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.cached_input_tokens), + output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.output_tokens), + reasoning_output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.reasoning_output_tokens), + total_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.total_tokens), duration_ms: completed.duration_ms, started_at, completed_at: Some(completed.completed_at), diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index ec42f0b611..312d90d31e 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -1340,8 +1340,14 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::TokenCount(token_count_event) => { - handle_token_count_event(conversation_id, event_turn_id, token_count_event, &outgoing) - .await; + handle_token_count_event( + conversation_id, + event_turn_id, + analytics_events_client.as_ref(), + token_count_event, + &outgoing, + ) + .await; } EventMsg::Error(ev) => { thread_watch_manager @@ -2194,6 +2200,7 @@ async fn handle_thread_rollback_failed( async fn handle_token_count_event( conversation_id: ThreadId, turn_id: String, + analytics_events_client: Option<&AnalyticsEventsClient>, token_count_event: TokenCountEvent, outgoing: &ThreadScopedOutgoingMessageSender, ) { @@ -2204,6 +2211,11 @@ async fn handle_token_count_event( turn_id, token_usage, }; + if let Some(analytics_events_client) = analytics_events_client { + analytics_events_client.track_notification( + ServerNotification::ThreadTokenUsageUpdated(notification.clone()), + ); + } outgoing .send_server_notification(ServerNotification::ThreadTokenUsageUpdated(notification)) .await; @@ -3989,6 +4001,7 @@ mod tests { handle_token_count_event( conversation_id, turn_id.clone(), + /*analytics_events_client*/ None, TokenCountEvent { info: Some(info), rate_limits: Some(rate_limits), @@ -4043,6 +4056,7 @@ mod tests { handle_token_count_event( conversation_id, turn_id.clone(), + /*analytics_events_client*/ None, TokenCountEvent { info: None, rate_limits: None, diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 5645a4d3a8..3381f7ad68 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -407,6 +407,11 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert!(event["event_params"]["started_at"].as_u64().is_some()); assert!(event["event_params"]["completed_at"].as_u64().is_some()); assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + assert_eq!(event["event_params"]["input_tokens"], 0); + assert_eq!(event["event_params"]["cached_input_tokens"], 0); + assert_eq!(event["event_params"]["output_tokens"], 0); + assert_eq!(event["event_params"]["reasoning_output_tokens"], 0); + assert_eq!(event["event_params"]["total_tokens"], 0); Ok(()) } diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index f33e6886f4..fadf8d9d9f 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -30,6 +30,7 @@ use crate::hook_runtime::record_pending_input; use crate::state::ActiveTurn; use crate::state::RunningTask; use crate::state::TaskKind; +use codex_analytics::TurnTokenUsageFact; use codex_login::AuthManager; use codex_models_manager::manager::ModelsManager; use codex_otel::SessionTelemetry; @@ -485,6 +486,13 @@ impl Session { - token_usage_at_turn_start.total_tokens) .max(0), }; + self.services + .analytics_events_client + .track_turn_token_usage(TurnTokenUsageFact { + turn_id: turn_context.sub_id.clone(), + thread_id: self.conversation_id.to_string(), + token_usage: turn_token_usage.clone(), + }); self.services.session_telemetry.histogram( TURN_TOKEN_USAGE_METRIC, turn_token_usage.total_tokens, From 8377066b8a78a5ed63dded0abcb4a4d0dc66641c Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 7 Apr 2026 12:20:29 -0700 Subject: [PATCH 3/5] [codex-analytics] add queued submission metadata --- .../analytics/src/analytics_client_tests.rs | 90 ++++++++++++++- .../schema/json/ClientRequest.json | 18 +++ .../codex_app_server_protocol.schemas.json | 18 +++ .../codex_app_server_protocol.v2.schemas.json | 18 +++ .../schema/json/v2/TurnStartParams.json | 18 +++ .../schema/typescript/SubmissionType.ts | 5 + .../schema/typescript/index.ts | 1 + .../schema/typescript/v2/TurnStartParams.ts | 4 + .../app-server-protocol/src/protocol/v2.rs | 7 ++ .../app-server/src/codex_message_processor.rs | 3 +- .../src/message_processor/tracing_tests.rs | 1 + .../app-server/tests/suite/v2/analytics.rs | 35 ++++++ .../app-server/tests/suite/v2/turn_start.rs | 107 ++++++++++++++++++ codex-rs/core/src/codex.rs | 42 ++++++- codex-rs/core/src/codex_tests.rs | 3 + codex-rs/core/src/guardian/review_session.rs | 1 + codex-rs/core/tests/common/test_codex.rs | 1 + codex-rs/core/tests/suite/apply_patch_cli.rs | 7 ++ codex-rs/core/tests/suite/approvals.rs | 1 + codex-rs/core/tests/suite/client.rs | 2 + codex-rs/core/tests/suite/code_mode.rs | 1 + .../tests/suite/collaboration_instructions.rs | 2 + codex-rs/core/tests/suite/compact.rs | 7 ++ codex-rs/core/tests/suite/exec_policy.rs | 2 + codex-rs/core/tests/suite/image_rollout.rs | 2 + codex-rs/core/tests/suite/items.rs | 5 + codex-rs/core/tests/suite/json_result.rs | 1 + codex-rs/core/tests/suite/live_reload.rs | 1 + codex-rs/core/tests/suite/model_switching.rs | 14 +++ .../core/tests/suite/model_visible_layout.rs | 5 + codex-rs/core/tests/suite/models_cache_ttl.rs | 1 + .../core/tests/suite/models_etag_responses.rs | 1 + codex-rs/core/tests/suite/personality.rs | 13 +++ codex-rs/core/tests/suite/prompt_caching.rs | 5 + codex-rs/core/tests/suite/remote_models.rs | 4 + .../core/tests/suite/request_permissions.rs | 1 + .../tests/suite/request_permissions_tool.rs | 1 + .../core/tests/suite/request_user_input.rs | 2 + codex-rs/core/tests/suite/rmcp_client.rs | 6 + .../tests/suite/safety_check_downgrade.rs | 4 + codex-rs/core/tests/suite/shell_snapshot.rs | 4 + codex-rs/core/tests/suite/skill_approval.rs | 1 + codex-rs/core/tests/suite/skills.rs | 1 + codex-rs/core/tests/suite/sqlite_state.rs | 1 + codex-rs/core/tests/suite/tool_harness.rs | 5 + codex-rs/core/tests/suite/tool_parallelism.rs | 2 + codex-rs/core/tests/suite/truncation.rs | 1 + codex-rs/core/tests/suite/unified_exec.rs | 26 +++++ codex-rs/core/tests/suite/user_shell_cmd.rs | 1 + codex-rs/core/tests/suite/view_image.rs | 14 +++ .../core/tests/suite/websocket_fallback.rs | 1 + codex-rs/exec/src/lib.rs | 1 + codex-rs/protocol/src/protocol.rs | 25 ++++ codex-rs/tui/src/app.rs | 2 + codex-rs/tui/src/app_command.rs | 6 + codex-rs/tui/src/app_server_session.rs | 3 + codex-rs/tui/src/chatwidget.rs | 17 ++- .../chatwidget/tests/composer_submission.rs | 23 ++++ 58 files changed, 587 insertions(+), 7 deletions(-) create mode 100644 codex-rs/app-server-protocol/schema/typescript/SubmissionType.ts diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index af7d957ea9..d026aeaac8 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -30,6 +30,7 @@ use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnSubmissionType; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; @@ -226,7 +227,7 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { turn_id: turn_id.to_string(), thread_id: "thread-2".to_string(), num_input_images: 1, - submission_type: None, + submission_type: Some(TurnSubmissionType::Default), model: "gpt-5".to_string(), model_provider: "openai".to_string(), sandbox_policy: SandboxPolicy::new_read_only_policy(), @@ -1048,7 +1049,7 @@ fn turn_event_serializes_expected_shape() { thread_id: "thread-2".to_string(), turn_id: "turn-2".to_string(), product_client_id: "codex-tui".to_string(), - submission_type: None, + submission_type: Some(TurnSubmissionType::Default), model: Some("gpt-5".to_string()), model_provider: "openai".to_string(), sandbox_policy: Some("read_only"), @@ -1094,7 +1095,7 @@ fn turn_event_serializes_expected_shape() { "thread_id": "thread-2", "turn_id": "turn-2", "product_client_id": "codex-tui", - "submission_type": null, + "submission_type": "default", "model": "gpt-5", "model_provider": "openai", "sandbox_policy": "read_only", @@ -1182,6 +1183,89 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } +#[tokio::test] +async fn queued_submission_type_emits_queued_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: CodexRuntimeMetadata { + codex_rs_version: "0.1.0".to_string(), + runtime_os: "macos".to_string(), + runtime_os_version: "15.3.1".to_string(), + runtime_arch: "aarch64".to_string(), + }, + rpc_transport: AppServerRpcTransport::Stdio, + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)), + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)), + }, + &mut out, + ) + .await; + let mut resolved_config = sample_turn_resolved_config("turn-2"); + resolved_config.submission_type = Some(TurnSubmissionType::Queued); + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + resolved_config, + ))), + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-2", "turn-2", + ))), + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["submission_type"], json!("queued")); +} + #[tokio::test] async fn turn_does_not_emit_without_required_prerequisites() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/app-server-protocol/schema/json/ClientRequest.json b/codex-rs/app-server-protocol/schema/json/ClientRequest.json index 9f1c46d800..f325c3825e 100644 --- a/codex-rs/app-server-protocol/schema/json/ClientRequest.json +++ b/codex-rs/app-server-protocol/schema/json/ClientRequest.json @@ -2510,6 +2510,13 @@ }, "type": "object" }, + "SubmissionType": { + "enum": [ + "prompt", + "prompt_queued" + ], + "type": "string" + }, "TextElement": { "properties": { "byteRange": { @@ -3278,6 +3285,17 @@ ], "description": "Override the service tier for this turn and subsequent turns." }, + "submissionType": { + "anyOf": [ + { + "$ref": "#/definitions/SubmissionType" + }, + { + "type": "null" + } + ], + "description": "Metadata describing how the prompt was submitted." + }, "summary": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index 80f2b226b7..65139a5ff6 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -12201,6 +12201,13 @@ } ] }, + "SubmissionType": { + "enum": [ + "prompt", + "prompt_queued" + ], + "type": "string" + }, "TerminalInteractionNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -14762,6 +14769,17 @@ ], "description": "Override the service tier for this turn and subsequent turns." }, + "submissionType": { + "anyOf": [ + { + "$ref": "#/definitions/v2/SubmissionType" + }, + { + "type": "null" + } + ], + "description": "Metadata describing how the prompt was submitted." + }, "summary": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index eb4e9d7b18..58931cee6d 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -10056,6 +10056,13 @@ } ] }, + "SubmissionType": { + "enum": [ + "prompt", + "prompt_queued" + ], + "type": "string" + }, "TerminalInteractionNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { @@ -12617,6 +12624,17 @@ ], "description": "Override the service tier for this turn and subsequent turns." }, + "submissionType": { + "anyOf": [ + { + "$ref": "#/definitions/SubmissionType" + }, + { + "type": "null" + } + ], + "description": "Metadata describing how the prompt was submitted." + }, "summary": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json index cad1d8b5bc..982cf96634 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartParams.json @@ -354,6 +354,13 @@ ], "type": "object" }, + "SubmissionType": { + "enum": [ + "prompt", + "prompt_queued" + ], + "type": "string" + }, "TextElement": { "properties": { "byteRange": { @@ -595,6 +602,17 @@ ], "description": "Override the service tier for this turn and subsequent turns." }, + "submissionType": { + "anyOf": [ + { + "$ref": "#/definitions/SubmissionType" + }, + { + "type": "null" + } + ], + "description": "Metadata describing how the prompt was submitted." + }, "summary": { "anyOf": [ { diff --git a/codex-rs/app-server-protocol/schema/typescript/SubmissionType.ts b/codex-rs/app-server-protocol/schema/typescript/SubmissionType.ts new file mode 100644 index 0000000000..7c32c07052 --- /dev/null +++ b/codex-rs/app-server-protocol/schema/typescript/SubmissionType.ts @@ -0,0 +1,5 @@ +// GENERATED CODE! DO NOT MODIFY BY HAND! + +// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. + +export type SubmissionType = "prompt" | "prompt_queued"; diff --git a/codex-rs/app-server-protocol/schema/typescript/index.ts b/codex-rs/app-server-protocol/schema/typescript/index.ts index 7ffc15e83d..a15d949c89 100644 --- a/codex-rs/app-server-protocol/schema/typescript/index.ts +++ b/codex-rs/app-server-protocol/schema/typescript/index.ts @@ -65,6 +65,7 @@ export type { ServiceTier } from "./ServiceTier"; export type { SessionSource } from "./SessionSource"; export type { Settings } from "./Settings"; export type { SubAgentSource } from "./SubAgentSource"; +export type { SubmissionType } from "./SubmissionType"; export type { ThreadId } from "./ThreadId"; export type { Tool } from "./Tool"; export type { Verbosity } from "./Verbosity"; diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartParams.ts b/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartParams.ts index 8f57a5e68b..9697838d9c 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartParams.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartParams.ts @@ -6,6 +6,7 @@ import type { Personality } from "../Personality"; import type { ReasoningEffort } from "../ReasoningEffort"; import type { ReasoningSummary } from "../ReasoningSummary"; import type { ServiceTier } from "../ServiceTier"; +import type { SubmissionType } from "../SubmissionType"; import type { JsonValue } from "../serde_json/JsonValue"; import type { ApprovalsReviewer } from "./ApprovalsReviewer"; import type { AskForApproval } from "./AskForApproval"; @@ -45,6 +46,9 @@ personality?: Personality | null, /** * this turn. */ outputSchema?: JsonValue | null, /** + * Metadata describing how the prompt was submitted. + */ +submissionType?: SubmissionType | null, /** * EXPERIMENTAL - Set a pre-set collaboration mode. * Takes precedence over model, reasoning_effort, and developer instructions if set. * diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 3562d9cda6..1836a1e08e 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -80,6 +80,7 @@ use codex_protocol::protocol::SkillMetadata as CoreSkillMetadata; use codex_protocol::protocol::SkillScope as CoreSkillScope; use codex_protocol::protocol::SkillToolDependency as CoreSkillToolDependency; use codex_protocol::protocol::SubAgentSource as CoreSubAgentSource; +use codex_protocol::protocol::SubmissionType; use codex_protocol::protocol::TokenUsage as CoreTokenUsage; use codex_protocol::protocol::TokenUsageInfo as CoreTokenUsageInfo; use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionGrantScope; @@ -4058,6 +4059,11 @@ pub struct TurnStartParams { #[ts(optional = nullable)] pub output_schema: Option, + /// Metadata describing how the prompt was submitted. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[ts(optional = nullable)] + pub submission_type: Option, + /// EXPERIMENTAL - Set a pre-set collaboration mode. /// Takes precedence over model, reasoning_effort, and developer instructions if set. /// @@ -8417,6 +8423,7 @@ mod tests { service_tier: None, effort: None, summary: None, + submission_type: None, output_schema: None, collaboration_mode: None, personality: None, diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 4c19fba551..636be2e981 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6647,9 +6647,10 @@ impl CodexMessageProcessor { .submit_core_op( &request_id, thread.as_ref(), - Op::UserInput { + Op::UserInputWithMetadata { items: mapped_items, final_output_json_schema: params.output_schema, + submission_type: params.submission_type, }, ) .await; diff --git a/codex-rs/app-server/src/message_processor/tracing_tests.rs b/codex-rs/app-server/src/message_processor/tracing_tests.rs index 2e8781606f..504b33cd8a 100644 --- a/codex-rs/app-server/src/message_processor/tracing_tests.rs +++ b/codex-rs/app-server/src/message_processor/tracing_tests.rs @@ -614,6 +614,7 @@ async fn turn_start_jsonrpc_span_parents_core_turn_spans() -> Result<()> { effort: None, summary: None, personality: None, + submission_type: None, output_schema: None, collaboration_mode: None, }, diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index babf6396d8..6c74b3dfb4 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -169,6 +169,41 @@ pub(crate) async fn wait_for_analytics_event( .await? } +pub(crate) async fn wait_for_analytics_turn_event( + server: &MockServer, + read_timeout: Duration, + turn_id: &str, +) -> Result { + timeout(read_timeout, async { + loop { + let Some(requests) = server.received_requests().await else { + tokio::time::sleep(Duration::from_millis(25)).await; + continue; + }; + for request in &requests { + if request.method != "POST" + || request.url.path() != "/codex/analytics-events/events" + { + continue; + } + let payload: Value = serde_json::from_slice(&request.body) + .map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?; + let Some(events) = payload["events"].as_array() else { + continue; + }; + if let Some(event) = events.iter().find(|event| { + event["event_type"] == "codex_turn_event" + && event["event_params"]["turn_id"] == turn_id + }) { + return Ok::(event.clone()); + } + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await? +} + pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> { let events = payload["events"] .as_array() diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 3381f7ad68..a503c3087c 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -55,6 +55,7 @@ use codex_protocol::config_types::Personality; use codex_protocol::config_types::ReasoningSummary; use codex_protocol::config_types::Settings; use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::SubmissionType; use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use core_test_support::responses; use core_test_support::skip_if_no_network; @@ -69,6 +70,7 @@ use tokio::time::timeout; use super::analytics::enable_analytics_capture; use super::analytics::wait_for_analytics_event; use super::analytics::wait_for_analytics_payload; +use super::analytics::wait_for_analytics_turn_event; #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); @@ -402,7 +404,9 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert_eq!(event["event_params"]["model"], "mock-model"); assert_eq!(event["event_params"]["model_provider"], "mock_provider"); assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); + assert_eq!(event["event_params"]["submission_type"], "default"); assert_eq!(event["event_params"]["num_input_images"], 1); + assert_eq!(event["event_params"]["is_first_turn"], true); assert_eq!(event["event_params"]["status"], "completed"); assert!(event["event_params"]["started_at"].as_u64().is_some()); assert!(event["event_params"]["completed_at"].as_u64().is_some()); @@ -479,6 +483,107 @@ async fn turn_start_does_not_track_turn_event_analytics_without_feature() -> Res payload.is_err(), "turn analytics should be gated off when general_analytics is disabled" ); + + Ok(()) +} + +#[tokio::test] +async fn turn_start_tracks_second_turn_as_not_first_and_queued_submission_analytics() -> Result<()> +{ + let responses = vec![ + create_final_assistant_message_sse_response("Done 1")?, + create_final_assistant_message_sse_response("Done 2")?, + ]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let first_turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "first turn".to_string(), + text_elements: Vec::new(), + }], + ..Default::default() + }) + .await?; + let first_turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(first_turn_req)), + ) + .await??; + let TurnStartResponse { turn: first_turn } = to_response::(first_turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let first_turn_event = + wait_for_analytics_turn_event(&server, DEFAULT_READ_TIMEOUT, &first_turn.id).await?; + assert_eq!( + first_turn_event["event_params"]["submission_type"], + "default" + ); + assert_eq!(first_turn_event["event_params"]["is_first_turn"], true); + + let second_turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id, + input: vec![V2UserInput::Text { + text: "second turn".to_string(), + text_elements: Vec::new(), + }], + submission_type: Some(SubmissionType::PromptQueued), + ..Default::default() + }) + .await?; + let second_turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(second_turn_req)), + ) + .await??; + let TurnStartResponse { turn: second_turn } = + to_response::(second_turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let second_turn_event = + wait_for_analytics_turn_event(&server, DEFAULT_READ_TIMEOUT, &second_turn.id).await?; + assert_eq!( + second_turn_event["event_params"]["submission_type"], + "queued" + ); + assert_eq!(second_turn_event["event_params"]["is_first_turn"], false); + Ok(()) } @@ -1636,6 +1741,7 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { summary: Some(ReasoningSummary::Auto), service_tier: None, personality: None, + submission_type: None, output_schema: None, collaboration_mode: None, }) @@ -1669,6 +1775,7 @@ async fn turn_start_updates_sandbox_and_cwd_between_turns_v2() -> Result<()> { summary: Some(ReasoningSummary::Auto), service_tier: None, personality: None, + submission_type: None, output_schema: None, collaboration_mode: None, }) diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index d8ab2e055c..6dddc247e0 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -52,6 +52,7 @@ use codex_analytics::AppInvocation; use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; use codex_analytics::TurnResolvedConfigFact; +use codex_analytics::TurnSubmissionType; use codex_analytics::build_track_events_context; use codex_app_server_protocol::McpServerElicitationRequest; use codex_app_server_protocol::McpServerElicitationRequestParams; @@ -118,6 +119,7 @@ use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::RolloutItem; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::SubmissionType; use codex_protocol::protocol::TurnAbortReason; use codex_protocol::protocol::TurnContextItem; use codex_protocol::protocol::TurnContextNetworkItem; @@ -883,6 +885,7 @@ pub(crate) struct TurnContext { pub(crate) current_date: Option, pub(crate) timezone: Option, pub(crate) app_server_client_name: Option, + pub(crate) submission_type: Option, pub(crate) developer_instructions: Option, pub(crate) compact_prompt: Option, pub(crate) user_instructions: Option, @@ -1001,6 +1004,7 @@ impl TurnContext { current_date: self.current_date.clone(), timezone: self.timezone.clone(), app_server_client_name: self.app_server_client_name.clone(), + submission_type: self.submission_type, developer_instructions: self.developer_instructions.clone(), compact_prompt: self.compact_prompt.clone(), user_instructions: self.user_instructions.clone(), @@ -1086,6 +1090,13 @@ impl TurnContext { } } +fn turn_submission_type(submission_type: SubmissionType) -> TurnSubmissionType { + match submission_type { + SubmissionType::Prompt => TurnSubmissionType::Default, + SubmissionType::PromptQueued => TurnSubmissionType::Queued, + } +} + fn local_time_context() -> (String, String) { match iana_time_zone::get_timezone() { Ok(timezone) => (Local::now().format("%Y-%m-%d").to_string(), timezone), @@ -1259,6 +1270,7 @@ pub(crate) struct SessionSettingsUpdate { pub(crate) personality: Option, pub(crate) app_server_client_name: Option, pub(crate) app_server_client_version: Option, + pub(crate) submission_type: Option, } pub(crate) struct AppServerClientMetadata { @@ -1427,6 +1439,7 @@ impl Session { network: Option, environment: Option>, sub_id: String, + submission_type: Option, js_repl: Arc, skills_outcome: Arc, ) -> TurnContext { @@ -1491,6 +1504,7 @@ impl Session { current_date: Some(current_date), timezone: Some(timezone), app_server_client_name: session_configuration.app_server_client_name.clone(), + submission_type, developer_instructions: session_configuration.developer_instructions.clone(), compact_prompt: session_configuration.compact_prompt.clone(), user_instructions: session_configuration.user_instructions.clone(), @@ -2501,6 +2515,7 @@ impl Session { sub_id, session_configuration, updates.final_output_json_schema, + updates.submission_type, sandbox_policy_changed, ) .await) @@ -2511,6 +2526,7 @@ impl Session { sub_id: String, session_configuration: SessionConfiguration, final_output_json_schema: Option>, + submission_type: Option, sandbox_policy_changed: bool, ) -> Arc { let per_turn_config = Self::build_per_turn_config(&session_configuration); @@ -2576,6 +2592,7 @@ impl Session { .map(StartedNetworkProxy::proxy), self.services.environment.clone(), sub_id, + submission_type, Arc::clone(&self.js_repl), skills_outcome, ); @@ -2688,6 +2705,7 @@ impl Session { sub_id, session_configuration, /*final_output_json_schema*/ None, + /*submission_type*/ None, /*sandbox_policy_changed*/ false, ) .await @@ -4622,7 +4640,7 @@ async fn submission_loop(sess: Arc, config: Arc, rx_sub: Receiv .await; false } - Op::UserInput { .. } | Op::UserTurn { .. } => { + Op::UserInput { .. } | Op::UserInputWithMetadata { .. } | Op::UserTurn { .. } => { handlers::user_input_or_turn(&sess, sub.id.clone(), sub.op).await; false } @@ -4880,6 +4898,7 @@ mod handlers { items, collaboration_mode, personality, + submission_type, } => { let collaboration_mode = collaboration_mode.or_else(|| { Some(CollaborationMode { @@ -4906,9 +4925,22 @@ mod handlers { personality, app_server_client_name: None, app_server_client_version: None, + submission_type, }, ) } + Op::UserInputWithMetadata { + items, + final_output_json_schema, + submission_type, + } => ( + items, + SessionSettingsUpdate { + final_output_json_schema: Some(final_output_json_schema), + submission_type, + ..Default::default() + }, + ), Op::UserInput { items, final_output_json_schema, @@ -5722,6 +5754,7 @@ async fn spawn_review_thread( current_date: parent_turn_context.current_date.clone(), timezone: parent_turn_context.timezone.clone(), app_server_client_name: parent_turn_context.app_server_client_name.clone(), + submission_type: None, developer_instructions: None, user_instructions: None, compact_prompt: parent_turn_context.compact_prompt.clone(), @@ -6360,7 +6393,12 @@ async fn track_turn_resolved_config_analytics( matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) }) .count(), - submission_type: None, + submission_type: Some( + turn_context + .submission_type + .map(turn_submission_type) + .unwrap_or(TurnSubmissionType::Default), + ), model: turn_context.model_info.slug.clone(), model_provider: turn_context.config.model_provider_id.clone(), sandbox_policy: turn_context.sandbox_policy.get().clone(), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index ae61cc0777..055b83b952 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -2806,6 +2806,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { /*network*/ None, Some(environment), "turn_id".to_string(), + /*submission_type*/ None, Arc::clone(&js_repl), skills_outcome, ); @@ -3174,6 +3175,7 @@ async fn user_turn_updates_approvals_reviewer() { final_output_json_schema: None, collaboration_mode: None, personality: config.personality, + submission_type: None, }, ) .await; @@ -3646,6 +3648,7 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( /*network*/ None, Some(environment), "turn_id".to_string(), + /*submission_type*/ None, Arc::clone(&js_repl), skills_outcome, )); diff --git a/codex-rs/core/src/guardian/review_session.rs b/codex-rs/core/src/guardian/review_session.rs index 368b812e56..fe472eb8be 100644 --- a/codex-rs/core/src/guardian/review_session.rs +++ b/codex-rs/core/src/guardian/review_session.rs @@ -523,6 +523,7 @@ async fn run_review_on_session( final_output_json_schema: Some(params.schema.clone()), collaboration_mode: None, personality: params.personality, + submission_type: None, }) .await }), diff --git a/codex-rs/core/tests/common/test_codex.rs b/codex-rs/core/tests/common/test_codex.rs index a6f3f0c72b..52f05e1caa 100644 --- a/codex-rs/core/tests/common/test_codex.rs +++ b/codex-rs/core/tests/common/test_codex.rs @@ -760,6 +760,7 @@ impl TestCodex { service_tier, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/apply_patch_cli.rs b/codex-rs/core/tests/suite/apply_patch_cli.rs index a8b429c0dd..c6a1b22970 100644 --- a/codex-rs/core/tests/suite/apply_patch_cli.rs +++ b/codex-rs/core/tests/suite/apply_patch_cli.rs @@ -368,6 +368,7 @@ async fn apply_patch_cli_move_without_content_change_has_no_turn_diff( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -978,6 +979,7 @@ async fn apply_patch_shell_command_heredoc_with_cd_emits_turn_diff() -> Result<( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1060,6 +1062,7 @@ async fn apply_patch_shell_command_failure_propagates_error_and_skips_diff() -> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1212,6 +1215,7 @@ async fn apply_patch_emits_turn_diff_event_with_unified_diff( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1277,6 +1281,7 @@ async fn apply_patch_turn_diff_for_rename_with_content_change( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1350,6 +1355,7 @@ async fn apply_patch_aggregates_diff_across_multiple_tool_calls() -> Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1423,6 +1429,7 @@ async fn apply_patch_aggregates_diff_preserves_success_after_failure() -> Result service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/approvals.rs b/codex-rs/core/tests/suite/approvals.rs index f7874eaf45..837c1c331c 100644 --- a/codex-rs/core/tests/suite/approvals.rs +++ b/codex-rs/core/tests/suite/approvals.rs @@ -597,6 +597,7 @@ async fn submit_turn( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/client.rs b/codex-rs/core/tests/suite/client.rs index 51ade5f464..0a966692e6 100644 --- a/codex-rs/core/tests/suite/client.rs +++ b/codex-rs/core/tests/suite/client.rs @@ -1669,6 +1669,7 @@ async fn user_turn_collaboration_mode_overrides_model_and_effort() -> anyhow::Re collaboration_mode: Some(collaboration_mode), final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; @@ -1783,6 +1784,7 @@ async fn user_turn_explicit_reasoning_summary_overrides_model_catalog_default() collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await .unwrap(); diff --git a/codex-rs/core/tests/suite/code_mode.rs b/codex-rs/core/tests/suite/code_mode.rs index 9ddc169a60..94b799a6af 100644 --- a/codex-rs/core/tests/suite/code_mode.rs +++ b/codex-rs/core/tests/suite/code_mode.rs @@ -2368,6 +2368,7 @@ text( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/collaboration_instructions.rs b/codex-rs/core/tests/suite/collaboration_instructions.rs index df8d6e4fe4..42ae909c85 100644 --- a/codex-rs/core/tests/suite/collaboration_instructions.rs +++ b/codex-rs/core/tests/suite/collaboration_instructions.rs @@ -191,6 +191,7 @@ async fn collaboration_instructions_added_on_user_turn() -> Result<()> { collaboration_mode: Some(collaboration_mode), final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -307,6 +308,7 @@ async fn user_turn_overrides_collaboration_instructions_after_override() -> Resu collaboration_mode: Some(turn_mode), final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; diff --git a/codex-rs/core/tests/suite/compact.rs b/codex-rs/core/tests/suite/compact.rs index df8fad124d..481b90e054 100644 --- a/codex-rs/core/tests/suite/compact.rs +++ b/codex-rs/core/tests/suite/compact.rs @@ -1668,6 +1668,7 @@ async fn auto_compact_runs_after_resume_when_token_usage_is_over_limit() { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .unwrap(); @@ -1759,6 +1760,7 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit first user turn"); @@ -1784,6 +1786,7 @@ async fn pre_sampling_compact_runs_on_switch_to_smaller_context_model() { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit second user turn"); @@ -1895,6 +1898,7 @@ async fn pre_sampling_compact_runs_after_resume_and_switch_to_smaller_model() { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit pre-resume turn"); @@ -1944,6 +1948,7 @@ async fn pre_sampling_compact_runs_after_resume_and_switch_to_smaller_model() { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit resumed user turn"); @@ -3149,6 +3154,7 @@ async fn snapshot_request_shape_pre_turn_compaction_strips_incoming_model_switch service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit first user turn"); @@ -3174,6 +3180,7 @@ async fn snapshot_request_shape_pre_turn_compaction_strips_incoming_model_switch service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await .expect("submit second user turn"); diff --git a/codex-rs/core/tests/suite/exec_policy.rs b/codex-rs/core/tests/suite/exec_policy.rs index fb055c970f..6de4141e6c 100644 --- a/codex-rs/core/tests/suite/exec_policy.rs +++ b/codex-rs/core/tests/suite/exec_policy.rs @@ -59,6 +59,7 @@ async fn submit_user_turn( service_tier: None, collaboration_mode, personality: None, + submission_type: None, }) .await?; Ok(()) @@ -140,6 +141,7 @@ async fn execpolicy_blocks_shell_invocation() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/image_rollout.rs b/codex-rs/core/tests/suite/image_rollout.rs index 8195bd0a86..0594e48f31 100644 --- a/codex-rs/core/tests/suite/image_rollout.rs +++ b/codex-rs/core/tests/suite/image_rollout.rs @@ -130,6 +130,7 @@ async fn copy_paste_local_image_persists_rollout_request_shape() -> anyhow::Resu service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -214,6 +215,7 @@ async fn drag_drop_image_persists_rollout_request_shape() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/items.rs b/codex-rs/core/tests/suite/items.rs index f949cd4b97..d5c748454d 100644 --- a/codex-rs/core/tests/suite/items.rs +++ b/codex-rs/core/tests/suite/items.rs @@ -530,6 +530,7 @@ async fn plan_mode_emits_plan_item_from_proposed_plan_block() -> anyhow::Result< service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; @@ -607,6 +608,7 @@ async fn plan_mode_strips_plan_from_agent_messages() -> anyhow::Result<()> { service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; @@ -716,6 +718,7 @@ async fn plan_mode_streaming_citations_are_stripped_across_added_deltas_and_done service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; @@ -903,6 +906,7 @@ async fn plan_mode_streaming_proposed_plan_tag_split_across_added_and_delta_is_p service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; @@ -1017,6 +1021,7 @@ async fn plan_mode_handles_missing_plan_close_tag() -> anyhow::Result<()> { service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/json_result.rs b/codex-rs/core/tests/suite/json_result.rs index 3b6f3e3f9c..027e34edbd 100644 --- a/codex-rs/core/tests/suite/json_result.rs +++ b/codex-rs/core/tests/suite/json_result.rs @@ -88,6 +88,7 @@ async fn codex_returns_json_result(model: String) -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/live_reload.rs b/codex-rs/core/tests/suite/live_reload.rs index 6ab001383f..a0d46f2072 100644 --- a/codex-rs/core/tests/suite/live_reload.rs +++ b/codex-rs/core/tests/suite/live_reload.rs @@ -69,6 +69,7 @@ async fn submit_skill_turn(test: &TestCodex, skill_path: PathBuf, prompt: &str) service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/model_switching.rs b/codex-rs/core/tests/suite/model_switching.rs index b14267af49..0ebd5e20b0 100644 --- a/codex-rs/core/tests/suite/model_switching.rs +++ b/codex-rs/core/tests/suite/model_switching.rs @@ -134,6 +134,7 @@ async fn model_change_appends_model_instructions_developer_message() -> Result<( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -171,6 +172,7 @@ async fn model_change_appends_model_instructions_developer_message() -> Result<( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -231,6 +233,7 @@ async fn model_and_personality_change_only_appends_model_instructions() -> Resul service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -268,6 +271,7 @@ async fn model_and_personality_change_only_appends_model_instructions() -> Resul service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -410,6 +414,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -431,6 +436,7 @@ async fn model_change_from_image_to_text_strips_prior_image_content() -> Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -539,6 +545,7 @@ async fn generated_image_is_replayed_for_image_capable_models() -> Result<()> { summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -560,6 +567,7 @@ async fn generated_image_is_replayed_for_image_capable_models() -> Result<()> { summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -671,6 +679,7 @@ async fn model_change_from_generated_image_to_text_preserves_prior_generated_ima summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -692,6 +701,7 @@ async fn model_change_from_generated_image_to_text_preserves_prior_generated_ima summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -805,6 +815,7 @@ async fn thread_rollback_after_generated_image_drops_entire_image_turn_history() summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -834,6 +845,7 @@ async fn thread_rollback_after_generated_image_drops_entire_image_turn_history() summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -989,6 +1001,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1048,6 +1061,7 @@ async fn model_switch_to_smaller_model_updates_token_context_window() -> Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/model_visible_layout.rs b/codex-rs/core/tests/suite/model_visible_layout.rs index 13e2819141..4284cdc099 100644 --- a/codex-rs/core/tests/suite/model_visible_layout.rs +++ b/codex-rs/core/tests/suite/model_visible_layout.rs @@ -129,6 +129,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |event| { @@ -153,6 +154,7 @@ async fn snapshot_model_visible_layout_turn_overrides() -> Result<()> { service_tier: None, collaboration_mode: None, personality: Some(Personality::Friendly), + submission_type: None, }) .await?; wait_for_event(&test.codex, |event| { @@ -232,6 +234,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |event| { @@ -256,6 +259,7 @@ async fn snapshot_model_visible_layout_cwd_change_does_not_refresh_agents() -> R service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |event| { @@ -366,6 +370,7 @@ async fn snapshot_model_visible_layout_resume_with_personality_change() -> Resul service_tier: None, collaboration_mode: None, personality: Some(Personality::Friendly), + submission_type: None, }) .await?; wait_for_event(&resumed.codex, |event| { diff --git a/codex-rs/core/tests/suite/models_cache_ttl.rs b/codex-rs/core/tests/suite/models_cache_ttl.rs index 94f2424db4..15cbc82d4b 100644 --- a/codex-rs/core/tests/suite/models_cache_ttl.rs +++ b/codex-rs/core/tests/suite/models_cache_ttl.rs @@ -105,6 +105,7 @@ async fn renews_cache_ttl_on_matching_models_etag() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/models_etag_responses.rs b/codex-rs/core/tests/suite/models_etag_responses.rs index daf3e89462..9fbbcdb93c 100644 --- a/codex-rs/core/tests/suite/models_etag_responses.rs +++ b/codex-rs/core/tests/suite/models_etag_responses.rs @@ -110,6 +110,7 @@ async fn refresh_models_on_models_etag_mismatch_and_avoid_duplicate_models_fetch service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/personality.rs b/codex-rs/core/tests/suite/personality.rs index 8a0cbdcb63..ea2507c766 100644 --- a/codex-rs/core/tests/suite/personality.rs +++ b/codex-rs/core/tests/suite/personality.rs @@ -111,6 +111,7 @@ async fn user_turn_personality_none_does_not_add_update_message() -> anyhow::Res service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -162,6 +163,7 @@ async fn config_personality_some_sets_instructions_template() -> anyhow::Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -220,6 +222,7 @@ async fn config_personality_none_sends_no_personality() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -284,6 +287,7 @@ async fn default_personality_is_pragmatic_without_config_toml() -> anyhow::Resul service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -336,6 +340,7 @@ async fn user_turn_personality_some_adds_update_message() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -374,6 +379,7 @@ async fn user_turn_personality_some_adds_update_message() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -441,6 +447,7 @@ async fn user_turn_personality_same_value_does_not_add_update_message() -> anyho service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -479,6 +486,7 @@ async fn user_turn_personality_same_value_does_not_add_update_message() -> anyho service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -559,6 +567,7 @@ async fn user_turn_personality_skips_if_feature_disabled() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -597,6 +606,7 @@ async fn user_turn_personality_skips_if_feature_disabled() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -714,6 +724,7 @@ async fn remote_model_friendly_personality_instructions_with_feature() -> anyhow service_tier: None, collaboration_mode: None, personality: Some(Personality::Friendly), + submission_type: None, }) .await?; @@ -833,6 +844,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -871,6 +883,7 @@ async fn user_turn_personality_remote_model_template_includes_update_message() - service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/prompt_caching.rs b/codex-rs/core/tests/suite/prompt_caching.rs index 4d34e691de..b62d793f58 100644 --- a/codex-rs/core/tests/suite/prompt_caching.rs +++ b/codex-rs/core/tests/suite/prompt_caching.rs @@ -712,6 +712,7 @@ async fn per_turn_overrides_keep_cached_prefix_and_key_constant() -> anyhow::Res collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -825,6 +826,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -846,6 +848,7 @@ async fn send_user_turn_with_no_changes_does_not_send_environment_context() -> a collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -951,6 +954,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; @@ -972,6 +976,7 @@ async fn send_user_turn_with_changes_sends_environment_context() -> anyhow::Resu collaboration_mode: None, final_output_json_schema: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&codex, |ev| matches!(ev, EventMsg::TurnComplete(_))).await; diff --git a/codex-rs/core/tests/suite/remote_models.rs b/codex-rs/core/tests/suite/remote_models.rs index 9559397b7e..5d08e69a29 100644 --- a/codex-rs/core/tests/suite/remote_models.rs +++ b/codex-rs/core/tests/suite/remote_models.rs @@ -184,6 +184,7 @@ async fn remote_models_long_model_slug_is_sent_with_high_reasoning() -> Result<( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -247,6 +248,7 @@ async fn namespaced_model_slug_uses_catalog_metadata_without_fallback_warning() service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -407,6 +409,7 @@ async fn remote_models_remote_model_uses_unified_exec() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -630,6 +633,7 @@ async fn remote_models_apply_remote_base_instructions() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/request_permissions.rs b/codex-rs/core/tests/suite/request_permissions.rs index 2cfd1cf6f7..6e25adb228 100644 --- a/codex-rs/core/tests/suite/request_permissions.rs +++ b/codex-rs/core/tests/suite/request_permissions.rs @@ -202,6 +202,7 @@ async fn submit_turn( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; Ok(()) diff --git a/codex-rs/core/tests/suite/request_permissions_tool.rs b/codex-rs/core/tests/suite/request_permissions_tool.rs index 14506f4a41..d44f47d931 100644 --- a/codex-rs/core/tests/suite/request_permissions_tool.rs +++ b/codex-rs/core/tests/suite/request_permissions_tool.rs @@ -153,6 +153,7 @@ async fn submit_turn( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; Ok(()) diff --git a/codex-rs/core/tests/suite/request_user_input.rs b/codex-rs/core/tests/suite/request_user_input.rs index 8e30b37c21..952c696653 100644 --- a/codex-rs/core/tests/suite/request_user_input.rs +++ b/codex-rs/core/tests/suite/request_user_input.rs @@ -153,6 +153,7 @@ async fn request_user_input_round_trip_for_mode(mode: ModeKind) -> anyhow::Resul }, }), personality: None, + submission_type: None, }) .await?; @@ -264,6 +265,7 @@ where service_tier: None, collaboration_mode: Some(collaboration_mode), personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/rmcp_client.rs b/codex-rs/core/tests/suite/rmcp_client.rs index 68c2bc0106..cb38a90919 100644 --- a/codex-rs/core/tests/suite/rmcp_client.rs +++ b/codex-rs/core/tests/suite/rmcp_client.rs @@ -138,6 +138,7 @@ async fn stdio_server_round_trip() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -309,6 +310,7 @@ async fn stdio_image_responses_round_trip() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -514,6 +516,7 @@ async fn stdio_image_responses_are_sanitized_for_text_only_model() -> anyhow::Re service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -630,6 +633,7 @@ async fn stdio_server_propagates_whitelisted_env_vars() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -793,6 +797,7 @@ async fn streamable_http_tool_call_round_trip() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1041,6 +1046,7 @@ async fn streamable_http_with_oauth_round_trip_impl() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/safety_check_downgrade.rs b/codex-rs/core/tests/suite/safety_check_downgrade.rs index 51a88ef16a..ef8c086275 100644 --- a/codex-rs/core/tests/suite/safety_check_downgrade.rs +++ b/codex-rs/core/tests/suite/safety_check_downgrade.rs @@ -53,6 +53,7 @@ async fn openai_model_header_mismatch_emits_warning_event_and_warning_item() -> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -152,6 +153,7 @@ async fn response_model_field_mismatch_emits_warning_when_header_matches_request service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -238,6 +240,7 @@ async fn openai_model_header_mismatch_only_emits_one_warning_per_turn() -> Resul service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -288,6 +291,7 @@ async fn openai_model_header_casing_only_mismatch_does_not_warn() -> Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/shell_snapshot.rs b/codex-rs/core/tests/suite/shell_snapshot.rs index 0e044f3922..960ebdb945 100644 --- a/codex-rs/core/tests/suite/shell_snapshot.rs +++ b/codex-rs/core/tests/suite/shell_snapshot.rs @@ -172,6 +172,7 @@ async fn run_snapshot_command_with_options( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -263,6 +264,7 @@ async fn run_shell_command_snapshot_with_options( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -334,6 +336,7 @@ async fn run_tool_turn_on_harness( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -568,6 +571,7 @@ async fn shell_command_snapshot_still_intercepts_apply_patch() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/skill_approval.rs b/codex-rs/core/tests/suite/skill_approval.rs index d77d736f94..15d096268a 100644 --- a/codex-rs/core/tests/suite/skill_approval.rs +++ b/codex-rs/core/tests/suite/skill_approval.rs @@ -59,6 +59,7 @@ async fn submit_turn_with_policies( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; Ok(()) diff --git a/codex-rs/core/tests/suite/skills.rs b/codex-rs/core/tests/suite/skills.rs index 388618b56a..690de66893 100644 --- a/codex-rs/core/tests/suite/skills.rs +++ b/codex-rs/core/tests/suite/skills.rs @@ -82,6 +82,7 @@ async fn user_turn_includes_skill_instructions() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/sqlite_state.rs b/codex-rs/core/tests/suite/sqlite_state.rs index f35152e185..e3a0a8e54c 100644 --- a/codex-rs/core/tests/suite/sqlite_state.rs +++ b/codex-rs/core/tests/suite/sqlite_state.rs @@ -404,6 +404,7 @@ async fn mcp_call_marks_thread_memory_mode_polluted_when_configured() -> Result< service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; wait_for_event(&test.codex, |event| { diff --git a/codex-rs/core/tests/suite/tool_harness.rs b/codex-rs/core/tests/suite/tool_harness.rs index 9594195a5b..296b5c4c09 100644 --- a/codex-rs/core/tests/suite/tool_harness.rs +++ b/codex-rs/core/tests/suite/tool_harness.rs @@ -93,6 +93,7 @@ async fn shell_tool_executes_command_and_streams_output() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -164,6 +165,7 @@ async fn update_plan_tool_emits_plan_update_event() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -245,6 +247,7 @@ async fn update_plan_tool_rejects_malformed_payload() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -341,6 +344,7 @@ async fn apply_patch_tool_executes_and_emits_patch_events() -> anyhow::Result<() service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -445,6 +449,7 @@ async fn apply_patch_reports_parse_diagnostics() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/tool_parallelism.rs b/codex-rs/core/tests/suite/tool_parallelism.rs index faff0b2e09..16b0a2edd7 100644 --- a/codex-rs/core/tests/suite/tool_parallelism.rs +++ b/codex-rs/core/tests/suite/tool_parallelism.rs @@ -50,6 +50,7 @@ async fn run_turn(test: &TestCodex, prompt: &str) -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -367,6 +368,7 @@ async fn shell_tools_start_before_response_completed_when_stream_delayed() -> an service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/truncation.rs b/codex-rs/core/tests/suite/truncation.rs index 3f4a9e5a64..5e4c980228 100644 --- a/codex-rs/core/tests/suite/truncation.rs +++ b/codex-rs/core/tests/suite/truncation.rs @@ -496,6 +496,7 @@ async fn mcp_image_output_preserves_image_and_no_text_summary() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/unified_exec.rs b/codex-rs/core/tests/suite/unified_exec.rs index 2f72277086..8039f39855 100644 --- a/codex-rs/core/tests/suite/unified_exec.rs +++ b/codex-rs/core/tests/suite/unified_exec.rs @@ -200,6 +200,7 @@ async fn unified_exec_intercepts_apply_patch_exec_command() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -334,6 +335,7 @@ async fn unified_exec_emits_exec_command_begin_event() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -417,6 +419,7 @@ async fn unified_exec_resolves_relative_workdir() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -503,6 +506,7 @@ async fn unified_exec_respects_workdir_override() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -601,6 +605,7 @@ async fn unified_exec_emits_exec_command_end_event() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -681,6 +686,7 @@ async fn unified_exec_emits_output_delta_for_exec_command() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -762,6 +768,7 @@ async fn unified_exec_full_lifecycle_with_background_end_event() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -897,6 +904,7 @@ async fn unified_exec_emits_terminal_interaction_for_write_stdin() -> Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1039,6 +1047,7 @@ async fn unified_exec_terminal_interaction_captures_delayed_output() -> Result<( service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1204,6 +1213,7 @@ async fn unified_exec_emits_one_begin_and_one_end_event() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1317,6 +1327,7 @@ async fn exec_command_reports_chunk_and_exit_metadata() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1440,6 +1451,7 @@ async fn unified_exec_defaults_to_pipe() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1535,6 +1547,7 @@ async fn unified_exec_can_enable_tty() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1621,6 +1634,7 @@ async fn unified_exec_respects_early_exit_notifications() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1757,6 +1771,7 @@ async fn write_stdin_returns_exit_metadata_and_clears_session() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1930,6 +1945,7 @@ async fn unified_exec_emits_end_event_when_session_dies_via_stdin() -> Result<() service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2012,6 +2028,7 @@ async fn unified_exec_keeps_long_running_session_after_turn_end() -> Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2105,6 +2122,7 @@ async fn unified_exec_interrupt_preserves_long_running_session() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2214,6 +2232,7 @@ async fn unified_exec_reuses_session_via_stdin() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2354,6 +2373,7 @@ PY service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; // This is a worst case scenario for the truncate logic. @@ -2473,6 +2493,7 @@ async fn unified_exec_timeout_and_followup_poll() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2574,6 +2595,7 @@ PY service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2675,6 +2697,7 @@ async fn unified_exec_runs_under_sandbox() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2784,6 +2807,7 @@ async fn unified_exec_python_prompt_under_seatbelt() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -2884,6 +2908,7 @@ async fn unified_exec_runs_on_all_platforms() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -3024,6 +3049,7 @@ async fn unified_exec_prunes_exited_sessions_first() -> Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/user_shell_cmd.rs b/codex-rs/core/tests/suite/user_shell_cmd.rs index f6abb5d621..c3ff787392 100644 --- a/codex-rs/core/tests/suite/user_shell_cmd.rs +++ b/codex-rs/core/tests/suite/user_shell_cmd.rs @@ -185,6 +185,7 @@ async fn user_shell_command_does_not_replace_active_turn() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/view_image.rs b/codex-rs/core/tests/suite/view_image.rs index f16d3c2b18..f9ac77748c 100644 --- a/codex-rs/core/tests/suite/view_image.rs +++ b/codex-rs/core/tests/suite/view_image.rs @@ -164,6 +164,7 @@ async fn user_turn_with_local_image_attaches_image() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -273,6 +274,7 @@ async fn view_image_tool_attaches_local_image() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -414,6 +416,7 @@ async fn view_image_tool_can_preserve_original_resolution_when_requested_on_gpt5 summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -520,6 +523,7 @@ async fn view_image_tool_errors_clearly_for_unsupported_detail_values() -> anyho summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -612,6 +616,7 @@ async fn view_image_tool_treats_null_detail_as_omitted() -> anyhow::Result<()> { summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -712,6 +717,7 @@ async fn view_image_tool_resizes_when_model_lacks_original_detail_support() -> a summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -823,6 +829,7 @@ async fn view_image_tool_does_not_force_original_resolution_with_capability_feat summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -925,6 +932,7 @@ await codex.emitImage(out); service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1045,6 +1053,7 @@ console.log(out.type); summary: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1138,6 +1147,7 @@ async fn view_image_tool_errors_when_path_is_directory() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1214,6 +1224,7 @@ async fn view_image_tool_errors_for_non_image_files() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1295,6 +1306,7 @@ async fn view_image_tool_errors_when_file_missing() -> anyhow::Result<()> { service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1425,6 +1437,7 @@ async fn view_image_tool_returns_unsupported_message_for_text_only_model() -> an service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; @@ -1500,6 +1513,7 @@ async fn replaces_invalid_local_image_after_bad_request() -> anyhow::Result<()> service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/core/tests/suite/websocket_fallback.rs b/codex-rs/core/tests/suite/websocket_fallback.rs index c55e72ec64..60db8b4bcf 100644 --- a/codex-rs/core/tests/suite/websocket_fallback.rs +++ b/codex-rs/core/tests/suite/websocket_fallback.rs @@ -165,6 +165,7 @@ async fn websocket_fallback_hides_first_websocket_retry_stream_error() -> Result service_tier: None, collaboration_mode: None, personality: None, + submission_type: None, }) .await?; diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 346e8e9e5e..d241ee3023 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -689,6 +689,7 @@ async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> { effort: default_effort, summary: None, personality: None, + submission_type: None, output_schema, collaboration_mode: None, }, diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 0f4cd84e2f..aca5c38b87 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -103,6 +103,13 @@ pub const REALTIME_CONVERSATION_OPEN_TAG: &str = ""; pub const REALTIME_CONVERSATION_CLOSE_TAG: &str = ""; pub const USER_MESSAGE_BEGIN: &str = "## My request for Codex:"; +#[derive(Debug, Clone, Copy, Deserialize, Serialize, PartialEq, Eq, JsonSchema, TS)] +#[serde(rename_all = "snake_case")] +pub enum SubmissionType { + Prompt, + PromptQueued, +} + /// Submission Queue Entry - requests from user #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct Submission { @@ -246,6 +253,19 @@ pub enum Op { final_output_json_schema: Option, }, + /// Same transport shape as [`Op::UserInput`], but preserves submission + /// metadata for analytics. + UserInputWithMetadata { + /// User input items, see `InputItem` + items: Vec, + /// Optional JSON Schema used to constrain the final assistant message for this turn. + #[serde(skip_serializing_if = "Option::is_none")] + final_output_json_schema: Option, + /// Metadata describing how the prompt was submitted. + #[serde(default, skip_serializing_if = "Option::is_none")] + submission_type: Option, + }, + /// Similar to [`Op::UserInput`], but contains additional context required /// for a turn of a [`crate::codex_thread::CodexThread`]. UserTurn { @@ -301,6 +321,10 @@ pub enum Op { /// Optional personality override for this turn. #[serde(skip_serializing_if = "Option::is_none")] personality: Option, + + /// Metadata describing how the prompt was submitted. + #[serde(default, skip_serializing_if = "Option::is_none")] + submission_type: Option, }, /// Inter-agent communication that should be recorded as assistant history @@ -581,6 +605,7 @@ impl Op { Self::RealtimeConversationText(_) => "realtime_conversation_text", Self::RealtimeConversationClose => "realtime_conversation_close", Self::UserInput { .. } => "user_input", + Self::UserInputWithMetadata { .. } => "user_input", Self::UserTurn { .. } => "user_turn", Self::InterAgentCommunication { .. } => "inter_agent_communication", Self::OverrideTurnContext { .. } => "override_turn_context", diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index cf42e85160..9437b6fc67 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -2234,6 +2234,7 @@ impl App { final_output_json_schema, collaboration_mode, personality, + submission_type, } => { let mut should_start_turn = true; if let Some(turn_id) = self.active_turn_id_for_thread(thread_id).await { @@ -2316,6 +2317,7 @@ impl App { *service_tier, collaboration_mode.clone(), *personality, + *submission_type, final_output_json_schema.clone(), ) .await?; diff --git a/codex-rs/tui/src/app_command.rs b/codex-rs/tui/src/app_command.rs index 0646cc297d..4a83d1ab95 100644 --- a/codex-rs/tui/src/app_command.rs +++ b/codex-rs/tui/src/app_command.rs @@ -17,6 +17,7 @@ use codex_protocol::protocol::Op; use codex_protocol::protocol::ReviewDecision; use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SubmissionType; use codex_protocol::request_permissions::RequestPermissionsResponse; use codex_protocol::request_user_input::RequestUserInputResponse; use codex_protocol::user_input::UserInput; @@ -51,6 +52,7 @@ pub(crate) enum AppCommandView<'a> { final_output_json_schema: &'a Option, collaboration_mode: &'a Option, personality: &'a Option, + submission_type: &'a Option, }, OverrideTurnContext { cwd: &'a Option, @@ -152,6 +154,7 @@ impl AppCommand { final_output_json_schema: Option, collaboration_mode: Option, personality: Option, + submission_type: Option, ) -> Self { Self(Op::UserTurn { items, @@ -166,6 +169,7 @@ impl AppCommand { final_output_json_schema, collaboration_mode, personality, + submission_type, }) } @@ -311,6 +315,7 @@ impl AppCommand { final_output_json_schema, collaboration_mode, personality, + submission_type, } => AppCommandView::UserTurn { items, cwd, @@ -324,6 +329,7 @@ impl AppCommand { final_output_json_schema, collaboration_mode, personality, + submission_type, }, Op::OverrideTurnContext { cwd, diff --git a/codex-rs/tui/src/app_server_session.rs b/codex-rs/tui/src/app_server_session.rs index d152e99f68..b4c287af7c 100644 --- a/codex-rs/tui/src/app_server_session.rs +++ b/codex-rs/tui/src/app_server_session.rs @@ -82,6 +82,7 @@ use codex_protocol::protocol::ReviewRequest; use codex_protocol::protocol::ReviewTarget as CoreReviewTarget; use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionNetworkProxyRuntime; +use codex_protocol::protocol::SubmissionType; use color_eyre::eyre::ContextCompat; use color_eyre::eyre::Result; use color_eyre::eyre::WrapErr; @@ -425,6 +426,7 @@ impl AppServerSession { service_tier: Option>, collaboration_mode: Option, personality: Option, + submission_type: Option, output_schema: Option, ) -> Result { let request_id = self.next_request_id(); @@ -443,6 +445,7 @@ impl AppServerSession { effort, summary, personality, + submission_type, output_schema, collaboration_mode, }, diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index f942308aa7..e087462f4d 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -193,6 +193,7 @@ use codex_protocol::protocol::ReviewTarget; use codex_protocol::protocol::SkillMetadata as ProtocolSkillMetadata; #[cfg(test)] use codex_protocol::protocol::StreamErrorEvent; +use codex_protocol::protocol::SubmissionType; use codex_protocol::protocol::TerminalInteractionEvent; use codex_protocol::protocol::TokenUsage; use codex_protocol::protocol::TokenUsageInfo; @@ -5567,6 +5568,14 @@ impl ChatWidget { } fn submit_user_message(&mut self, user_message: UserMessage) { + self.submit_user_message_with_type(user_message, /*submission_type*/ None); + } + + fn submit_user_message_with_type( + &mut self, + user_message: UserMessage, + submission_type: Option, + ) { if !self.is_session_configured() { tracing::warn!("cannot submit user message before session is configured; queueing"); self.queued_user_messages.push_front(user_message); @@ -5782,6 +5791,7 @@ impl ChatWidget { /*final_output_json_schema*/ None, collaboration_mode, personality, + submission_type, ); if !self.submit_op(op) { @@ -7243,8 +7253,13 @@ impl ChatWidget { if self.bottom_pane.is_task_running() { return; } + let submission_type = if self.rejected_steers_queue.is_empty() { + Some(SubmissionType::PromptQueued) + } else { + None + }; if let Some(user_message) = self.pop_next_queued_user_message() { - self.submit_user_message(user_message); + self.submit_user_message_with_type(user_message, submission_type); } // Update the list to reflect the remaining queued messages (if any). self.refresh_pending_input_preview(); diff --git a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs index 1bae5d2aa8..f272ceb81e 100644 --- a/codex-rs/tui/src/chatwidget/tests/composer_submission.rs +++ b/codex-rs/tui/src/chatwidget/tests/composer_submission.rs @@ -642,6 +642,29 @@ async fn interrupted_turn_restore_keeps_active_mode_for_resubmission() { assert_eq!(chat.active_collaboration_mode_kind(), expected_mode); } +#[tokio::test] +async fn queued_user_message_marks_prompt_queued_submission_type() { + let (mut chat, _rx, mut op_rx) = make_chatwidget_manual(Some("gpt-5")).await; + chat.thread_id = Some(ThreadId::new()); + chat.queued_user_messages.push_back(UserMessage { + text: "queued follow up".to_string(), + local_images: Vec::new(), + remote_image_urls: Vec::new(), + text_elements: Vec::new(), + mention_bindings: Vec::new(), + }); + + chat.maybe_send_next_queued_input(); + + match next_submit_op(&mut op_rx) { + Op::UserTurn { + submission_type: Some(SubmissionType::PromptQueued), + .. + } => {} + other => panic!("expected queued prompt submission, got {other:?}"), + } +} + #[tokio::test] async fn remap_placeholders_uses_attachment_labels() { let placeholder_one = "[Image #1]"; From 55bbfccf6f47d18c95b4042551aa6b01334aeaee Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 7 Apr 2026 12:20:29 -0700 Subject: [PATCH 4/5] [codex-analytics] add steering metadata --- .../analytics/src/analytics_client_tests.rs | 402 +++++++++++++++++- codex-rs/analytics/src/client.rs | 11 + codex-rs/analytics/src/events.rs | 42 ++ codex-rs/analytics/src/facts.rs | 33 ++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 75 +++- .../app-server/tests/suite/v2/turn_steer.rs | 77 ++-- codex-rs/core/src/codex.rs | 186 +++++++- 8 files changed, 781 insertions(+), 48 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index d026aeaac8..ffa1aa489c 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -7,6 +7,7 @@ use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventRequest; +use crate::events::CodexTurnSteerEventRequest; use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; @@ -14,11 +15,13 @@ use crate::events::TrackEventRequest; use crate::events::codex_app_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; +use crate::events::codex_turn_steer_event_params; use crate::events::subagent_thread_started_event_request; use crate::facts::AnalyticsFact; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; +use crate::facts::CodexTurnSteerEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::InvocationType; use crate::facts::PluginState; @@ -30,6 +33,9 @@ use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnSteerInput; +use crate::facts::TurnSteerRejectionReason; +use crate::facts::TurnSteerResult; use crate::facts::TurnSubmissionType; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; @@ -243,6 +249,25 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { } } +fn sample_app_server_client_metadata() -> CodexAppServerClientMetadata { + CodexAppServerClientMetadata { + product_client_id: "codex-tui".to_string(), + client_name: Some("codex-tui".to_string()), + client_version: Some("1.0.0".to_string()), + rpc_transport: AppServerRpcTransport::Stdio, + experimental_api_enabled: None, + } +} + +fn sample_runtime_metadata() -> CodexRuntimeMetadata { + CodexRuntimeMetadata { + codex_rs_version: "0.1.0".to_string(), + runtime_os: "macos".to_string(), + runtime_os_version: "15.3.1".to_string(), + runtime_arch: "aarch64".to_string(), + } +} + async fn ingest_turn_prerequisites( reducer: &mut AnalyticsReducer, out: &mut Vec, @@ -1065,7 +1090,7 @@ fn turn_event_serializes_expected_shape() { is_first_turn: true, status: Some(TurnStatus::Completed), turn_error: None, - steer_count: None, + steer_count: Some(0), total_tool_call_count: None, shell_command_count: None, file_change_count: None, @@ -1111,7 +1136,7 @@ fn turn_event_serializes_expected_shape() { "is_first_turn": true, "status": "completed", "turn_error": null, - "steer_count": null, + "steer_count": 0, "total_tool_call_count": null, "shell_command_count": null, "file_change_count": null, @@ -1133,6 +1158,280 @@ fn turn_event_serializes_expected_shape() { ); } +#[test] +fn turn_steer_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }; + let event = TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest { + event_type: "codex_turn_steer_event", + event_params: codex_turn_steer_event_params( + sample_app_server_client_metadata(), + sample_runtime_metadata(), + &tracking, + CodexTurnSteerEvent { + expected_turn_id: Some("turn-2".to_string()), + accepted_turn_id: Some("turn-2".to_string()), + num_input_images: 2, + result: TurnSteerResult::Accepted, + rejection_reason: None, + created_at: 1_716_000_123, + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize turn steer event"); + + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2")); + assert_eq!(payload["event_params"]["accepted_turn_id"], json!("turn-2")); + assert_eq!( + payload["event_params"]["app_server_client"], + json!({ + "product_client_id": "codex-tui", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": null, + }) + ); + assert_eq!( + payload["event_params"]["runtime"], + json!({ + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64", + }) + ); + assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["num_input_images"], json!(2)); + assert_eq!(payload["event_params"]["result"], json!("accepted")); + assert_eq!(payload["event_params"]["rejection_reason"], json!(null)); + assert_eq!(payload["event_params"]["started_at"], json!(1_716_000_123)); + assert!(payload["event_params"].get("created_at").is_none()); +} + +#[test] +fn rejected_turn_steer_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-3".to_string(), + turn_id: "turn-3".to_string(), + }; + let event = TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest { + event_type: "codex_turn_steer_event", + event_params: codex_turn_steer_event_params( + sample_app_server_client_metadata(), + sample_runtime_metadata(), + &tracking, + CodexTurnSteerEvent { + expected_turn_id: Some("turn-expected".to_string()), + accepted_turn_id: None, + num_input_images: 1, + result: TurnSteerResult::Rejected, + rejection_reason: Some(TurnSteerRejectionReason::ExpectedTurnMismatch), + created_at: 1_716_000_124, + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize rejected turn steer event"); + + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-3")); + assert_eq!( + payload["event_params"]["expected_turn_id"], + json!("turn-expected") + ); + assert_eq!(payload["event_params"]["accepted_turn_id"], json!(null)); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + json!("0.1.0") + ); + assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!(payload["event_params"]["result"], json!("rejected")); + assert_eq!( + payload["event_params"]["rejection_reason"], + json!("expected_turn_mismatch") + ); + assert_eq!(payload["event_params"]["started_at"], json!(1_716_000_124)); + assert!(payload["event_params"].get("created_at").is_none()); +} + +#[tokio::test] +async fn turn_steer_uses_connection_metadata() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ false, + /*include_token_usage*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: Some("turn-2".to_string()), + accepted_turn_id: Some("turn-2".to_string()), + num_input_images: 2, + result: TurnSteerResult::Accepted, + rejection_reason: None, + created_at: 1_716_000_125, + }, + })), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event"); + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!( + payload["event_params"]["app_server_client"], + json!({ + "product_client_id": "codex-tui", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": null, + }) + ); + assert_eq!( + payload["event_params"]["runtime"], + json!({ + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64", + }) + ); + assert!(payload["event_params"].get("product_client_id").is_none()); +} + +#[tokio::test] +async fn rejected_turn_steer_uses_thread_connection_metadata() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: sample_runtime_metadata(), + rpc_transport: AppServerRpcTransport::Stdio, + }, + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_thread_start_response( + "thread-2", /*ephemeral*/ false, "gpt-5", + )), + }, + &mut out, + ) + .await; + out.clear(); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: String::new(), + thread_id: "thread-2".to_string(), + turn_id: String::new(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: None, + accepted_turn_id: None, + num_input_images: 1, + result: TurnSteerResult::Rejected, + rejection_reason: Some(TurnSteerRejectionReason::NoActiveTurn), + created_at: 1_716_000_126, + }, + })), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event"); + assert_eq!(payload["event_type"], json!("codex_turn_steer_event")); + assert_eq!( + payload["event_params"]["app_server_client"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!( + payload["event_params"]["runtime"]["codex_rs_version"], + json!("0.1.0") + ); + assert_eq!(payload["event_params"]["result"], json!("rejected")); + assert_eq!( + payload["event_params"]["rejection_reason"], + json!("no_active_turn") + ); +} + +#[tokio::test] +async fn turn_steer_does_not_emit_without_connection_metadata() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: Some("turn-2".to_string()), + accepted_turn_id: None, + num_input_images: 1, + result: TurnSteerResult::Rejected, + rejection_reason: Some(TurnSteerRejectionReason::NoActiveTurn), + created_at: 1_716_000_126, + }, + })), + &mut out, + ) + .await; + + assert!(out.is_empty()); +} + #[tokio::test] async fn turn_lifecycle_emits_turn_event() { let mut reducer = AnalyticsReducer::default(); @@ -1170,6 +1469,7 @@ async fn turn_lifecycle_emits_turn_event() { ); assert_eq!(payload["event_params"]["num_input_images"], json!(1)); assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["steer_count"], json!(0)); assert_eq!(payload["event_params"]["started_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); @@ -1183,6 +1483,104 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } +#[tokio::test] +async fn accepted_steers_increment_turn_steer_count() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: Some("turn-2".to_string()), + accepted_turn_id: Some("turn-2".to_string()), + num_input_images: 0, + result: TurnSteerResult::Accepted, + rejection_reason: None, + created_at: 1, + }, + })), + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: None, + accepted_turn_id: None, + num_input_images: 0, + result: TurnSteerResult::Rejected, + rejection_reason: Some(TurnSteerRejectionReason::NoActiveTurn), + created_at: 2, + }, + })), + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer(TurnSteerInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }, + turn_steer: CodexTurnSteerEvent { + expected_turn_id: Some("turn-2".to_string()), + accepted_turn_id: Some("turn-2".to_string()), + num_input_images: 1, + result: TurnSteerResult::Accepted, + rejection_reason: None, + created_at: 3, + }, + })), + &mut out, + ) + .await; + + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let turn_event = out + .iter() + .find(|event| matches!(event, TrackEventRequest::TurnEvent(_))) + .expect("turn event should be emitted"); + let payload = serde_json::to_value(turn_event).expect("serialize turn event"); + assert_eq!(payload["event_params"]["steer_count"], json!(2)); +} + #[tokio::test] async fn queued_submission_type_emits_queued_turn_event() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 41802bdb0e..5a5d2f30b5 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -6,6 +6,7 @@ use crate::facts::AnalyticsFact; use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; +use crate::facts::CodexTurnSteerEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -14,6 +15,7 @@ use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnSteerInput; use crate::facts::TurnTokenUsageFact; use crate::reducer::AnalyticsReducer; use codex_app_server_protocol::ClientRequest; @@ -203,6 +205,15 @@ impl AnalyticsEventsClient { ))); } + pub fn track_turn_steer(&self, tracking: TrackEventsContext, turn_steer: CodexTurnSteerEvent) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::TurnSteer( + TurnSteerInput { + tracking, + turn_steer, + }, + ))); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 48914d9de9..23e6cccbe4 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1,9 +1,12 @@ use crate::facts::AppInvocation; +use crate::facts::CodexTurnSteerEvent; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; use crate::facts::TrackEventsContext; use crate::facts::TurnStatus; +use crate::facts::TurnSteerRejectionReason; +use crate::facts::TurnSteerResult; use crate::facts::TurnSubmissionType; use codex_app_server_protocol::CodexErrorInfo; use codex_login::default_client::originator; @@ -41,6 +44,7 @@ pub(crate) enum TrackEventRequest { AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), TurnEvent(Box), + TurnSteer(CodexTurnSteerEventRequest), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -172,6 +176,25 @@ pub(crate) struct CodexTurnEventRequest { pub(crate) event_params: CodexTurnEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexTurnSteerEventParams { + pub(crate) thread_id: String, + pub(crate) expected_turn_id: Option, + pub(crate) accepted_turn_id: Option, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) num_input_images: usize, + pub(crate) result: TurnSteerResult, + pub(crate) rejection_reason: Option, + pub(crate) started_at: u64, +} + +#[derive(Serialize)] +pub(crate) struct CodexTurnSteerEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexTurnSteerEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexPluginMetadata { pub(crate) plugin_id: Option, @@ -263,6 +286,25 @@ pub(crate) fn codex_plugin_used_metadata( } } +pub(crate) fn codex_turn_steer_event_params( + app_server_client: CodexAppServerClientMetadata, + runtime: CodexRuntimeMetadata, + tracking: &TrackEventsContext, + turn_steer: CodexTurnSteerEvent, +) -> CodexTurnSteerEventParams { + CodexTurnSteerEventParams { + thread_id: tracking.thread_id.clone(), + expected_turn_id: turn_steer.expected_turn_id, + accepted_turn_id: turn_steer.accepted_turn_id, + app_server_client, + runtime, + num_input_images: turn_steer.num_input_images, + result: turn_steer.result, + rejection_reason: turn_steer.rejection_reason, + started_at: turn_steer.created_at, + } +} + pub(crate) fn thread_source_name(thread_source: &SessionSource) -> Option<&'static str> { match thread_source { SessionSource::Cli | SessionSource::VSCode | SessionSource::Exec => Some("user"), diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index ed5d96bb25..6c11b06c28 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -81,6 +81,33 @@ pub enum TurnStatus { Interrupted, } +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSteerResult { + Accepted, + Rejected, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSteerRejectionReason { + NoActiveTurn, + ExpectedTurnMismatch, + NonSteerableReview, + NonSteerableCompact, + EmptyInput, +} + +#[derive(Clone)] +pub struct CodexTurnSteerEvent { + pub expected_turn_id: Option, + pub accepted_turn_id: Option, + pub num_input_images: usize, + pub result: TurnSteerResult, + pub rejection_reason: Option, + pub created_at: u64, +} + #[derive(Clone, Debug)] pub struct SkillInvocation { pub skill_name: String, @@ -142,6 +169,7 @@ pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), TurnResolvedConfig(Box), TurnTokenUsage(Box), + TurnSteer(TurnSteerInput), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), @@ -149,6 +177,11 @@ pub(crate) enum CustomAnalyticsFact { PluginStateChanged(PluginStateChangedInput), } +pub(crate) struct TurnSteerInput { + pub tracking: TrackEventsContext, + pub turn_steer: CodexTurnSteerEvent, +} + pub(crate) struct SkillInvokedInput { pub tracking: TrackEventsContext, pub invocations: Vec, diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 5a4630ff47..b3e829ea03 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -6,12 +6,15 @@ mod reducer; pub use client::AnalyticsEventsClient; pub use events::AppServerRpcTransport; pub use facts::AppInvocation; +pub use facts::CodexTurnSteerEvent; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; pub use facts::TrackEventsContext; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; +pub use facts::TurnSteerRejectionReason; +pub use facts::TurnSteerResult; pub use facts::TurnSubmissionType; pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index e24900ef04..d6f3ebd5e5 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -7,6 +7,7 @@ use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventParams; use crate::events::CodexTurnEventRequest; +use crate::events::CodexTurnSteerEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializationMode; @@ -16,20 +17,25 @@ use crate::events::TrackEventRequest; use crate::events::codex_app_metadata; use crate::events::codex_plugin_metadata; use crate::events::codex_plugin_used_metadata; +use crate::events::codex_turn_steer_event_params; use crate::events::plugin_state_event_type; use crate::events::subagent_thread_started_event_request; use crate::events::thread_source_name; use crate::facts::AnalyticsFact; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; +use crate::facts::CodexTurnSteerEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; +use crate::facts::TurnSteerInput; +use crate::facts::TurnSteerResult; use crate::facts::TurnTokenUsageFact; use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; @@ -57,6 +63,7 @@ pub(crate) struct AnalyticsReducer { requests: HashMap<(u64, RequestId), RequestState>, turns: HashMap, connections: HashMap, + thread_connections: HashMap, } struct ConnectionState { @@ -89,6 +96,7 @@ struct TurnState { started_at: Option, token_usage: Option, completed: Option, + steer_count: usize, } impl AnalyticsReducer { @@ -135,6 +143,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnTokenUsage(input) => { self.ingest_turn_token_usage(*input, out); } + CustomAnalyticsFact::TurnSteer(input) => { + self.ingest_turn_steer(input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -198,6 +209,8 @@ impl AnalyticsReducer { let ClientRequest::TurnStart { params, .. } = request else { return; }; + self.thread_connections + .insert(params.thread_id.clone(), connection_id); self.requests.insert( (connection_id, request_id), RequestState::TurnStart(PendingTurnStartState { @@ -229,6 +242,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.thread_id = Some(thread_id); turn_state.num_input_images = Some(num_input_images); @@ -250,6 +264,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.thread_id = Some(input.thread_id); turn_state.token_usage = Some(input.token_usage); @@ -402,6 +417,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.connection_id = Some(connection_id); turn_state.thread_id = Some(pending_request.thread_id); @@ -427,6 +443,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.started_at = notification .turn @@ -445,6 +462,7 @@ impl AnalyticsReducer { started_at: None, token_usage: None, completed: None, + steer_count: 0, }); turn_state.completed = Some(CompletedTurnState { status: analytics_turn_status(notification.turn.status), @@ -478,6 +496,8 @@ impl AnalyticsReducer { out: &mut Vec, ) { let thread_source: SessionSource = thread.source.into(); + self.thread_connections + .insert(thread.id.clone(), connection_id); let Some(connection_state) = self.connections.get(&connection_id) else { return; }; @@ -500,6 +520,59 @@ impl AnalyticsReducer { )); } + fn ingest_turn_steer(&mut self, input: TurnSteerInput, out: &mut Vec) { + let TurnSteerInput { + tracking, + turn_steer, + } = input; + if matches!(turn_steer.result, TurnSteerResult::Accepted) + && let Some(accepted_turn_id) = turn_steer.accepted_turn_id.as_ref() + && let Some(turn_state) = self.turns.get_mut(accepted_turn_id) + { + turn_state.steer_count += 1; + } + let Some((app_server_client, runtime)) = + self.connection_metadata_for_turn_steer(&tracking, &turn_steer) + else { + return; + }; + out.push(TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest { + event_type: "codex_turn_steer_event", + event_params: codex_turn_steer_event_params( + app_server_client, + runtime, + &tracking, + turn_steer, + ), + })); + } + + fn connection_metadata_for_turn_steer( + &self, + tracking: &TrackEventsContext, + turn_steer: &CodexTurnSteerEvent, + ) -> Option<(CodexAppServerClientMetadata, CodexRuntimeMetadata)> { + let turn_connection_id = turn_steer + .accepted_turn_id + .as_ref() + .or(turn_steer.expected_turn_id.as_ref()) + .or(Some(&tracking.turn_id)) + .and_then(|turn_id| self.turns.get(turn_id)) + .and_then(|turn_state| turn_state.connection_id); + + let connection_id = turn_connection_id + .or_else(|| self.thread_connections.get(&tracking.thread_id).copied()); + + connection_id + .and_then(|connection_id| self.connections.get(&connection_id)) + .map(|connection_state| { + ( + connection_state.app_server_client.clone(), + connection_state.runtime.clone(), + ) + }) + } + fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec) { let Some(turn_state) = self.turns.get(turn_id) else { return; @@ -585,7 +658,7 @@ fn codex_turn_event_params( is_first_turn, status: completed.status, turn_error: completed.turn_error, - steer_count: None, + steer_count: Some(turn_state.steer_count), total_tool_call_count: None, shell_command_count: None, file_change_count: None, diff --git a/codex-rs/app-server/tests/suite/v2/turn_steer.rs b/codex-rs/app-server/tests/suite/v2/turn_steer.rs index 5d1b3cc22e..76da7efb78 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_steer.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_steer.rs @@ -6,6 +6,7 @@ use app_test_support::create_mock_responses_server_sequence; use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::JSONRPCError; @@ -23,6 +24,9 @@ use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS; use tempfile::TempDir; use tokio::time::timeout; +use super::analytics::enable_analytics_capture; +use super::analytics::wait_for_analytics_event; + const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); #[tokio::test] @@ -32,7 +36,12 @@ async fn turn_steer_requires_active_turn() -> Result<()> { std::fs::create_dir(&codex_home)?; let server = create_mock_responses_server_sequence(vec![]).await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -52,7 +61,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> { let steer_req = mcp .send_turn_steer_request(TurnSteerParams { - thread_id: thread.id, + thread_id: thread.id.clone(), input: vec![V2UserInput::Text { text: "steer".to_string(), text_elements: Vec::new(), @@ -67,6 +76,21 @@ async fn turn_steer_requires_active_turn() -> Result<()> { .await??; assert_eq!(steer_err.error.code, -32600); + let event = + wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["result"], "rejected"); + assert_eq!(event["event_params"]["num_input_images"], 0); + assert_eq!( + event["event_params"]["expected_turn_id"], + "turn-does-not-exist" + ); + assert_eq!( + event["event_params"]["accepted_turn_id"], + serde_json::Value::Null + ); + assert_eq!(event["event_params"]["rejection_reason"], "no_active_turn"); + Ok(()) } @@ -95,7 +119,12 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> { "call_sleep", )?]) .await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -198,7 +227,12 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { "call_sleep", )?]) .await; - create_config_toml(&codex_home, &server.uri())?; + write_mock_responses_config_toml_with_chatgpt_base_url( + &codex_home, + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, &codex_home).await?; let mut mcp = McpProcess::new(&codex_home).await?; timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; @@ -258,31 +292,20 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> { let steer: TurnSteerResponse = to_response::(steer_resp)?; assert_eq!(steer.turn_id, turn.id); + let event = + wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["result"], "accepted"); + assert_eq!(event["event_params"]["num_input_images"], 0); + assert_eq!(event["event_params"]["expected_turn_id"], turn.id); + assert_eq!(event["event_params"]["accepted_turn_id"], turn.id); + assert_eq!( + event["event_params"]["rejection_reason"], + serde_json::Value::Null + ); + mcp.interrupt_turn_and_wait_for_aborted(thread.id, steer.turn_id, DEFAULT_READ_TIMEOUT) .await?; Ok(()) } - -fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> { - let config_toml = codex_home.join("config.toml"); - std::fs::write( - config_toml, - format!( - r#" -model = "mock-model" -approval_policy = "never" -sandbox_mode = "danger-full-access" - -model_provider = "mock_provider" - -[model_providers.mock_provider] -name = "Mock provider for test" -base_url = "{server_uri}/v1" -wire_api = "responses" -request_max_retries = 0 -stream_max_retries = 0 -"# - ), - ) -} diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 6dddc247e0..81bc3d7141 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -49,9 +49,13 @@ use chrono::Local; use chrono::Utc; use codex_analytics::AnalyticsEventsClient; use codex_analytics::AppInvocation; +use codex_analytics::CodexTurnSteerEvent; use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; +use codex_analytics::TrackEventsContext; use codex_analytics::TurnResolvedConfigFact; +use codex_analytics::TurnSteerRejectionReason; +use codex_analytics::TurnSteerResult; use codex_analytics::TurnSubmissionType; use codex_analytics::build_track_events_context; use codex_app_server_protocol::McpServerElicitationRequest; @@ -240,6 +244,32 @@ impl SteerInputError { }, } } + + fn to_turn_steer_rejection_reason(&self) -> TurnSteerRejectionReason { + match self { + Self::NoActiveTurn(_) => TurnSteerRejectionReason::NoActiveTurn, + Self::ExpectedTurnMismatch { .. } => TurnSteerRejectionReason::ExpectedTurnMismatch, + Self::ActiveTurnNotSteerable { turn_kind } => match turn_kind { + NonSteerableTurnKind::Review => TurnSteerRejectionReason::NonSteerableReview, + NonSteerableTurnKind::Compact => TurnSteerRejectionReason::NonSteerableCompact, + }, + Self::EmptyInput => TurnSteerRejectionReason::EmptyInput, + } + } +} + +struct AcceptedSteerInput { + turn_id: String, + tracking: TrackEventsContext, + expected_turn_id: String, + num_input_images: usize, +} + +struct RejectedSteerInput { + error: SteerInputError, + tracking: TrackEventsContext, + expected_turn_id: Option, + num_input_images: usize, } /// Notes from the previous real user turn. @@ -4096,47 +4126,163 @@ impl Session { input: Vec, expected_turn_id: Option<&str>, ) -> Result { - if input.is_empty() { - return Err(SteerInputError::EmptyInput); + let created_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + match self + .try_append_input_to_active_turn(input, expected_turn_id) + .await + { + Ok(accepted) => { + self.services.analytics_events_client.track_turn_steer( + accepted.tracking, + CodexTurnSteerEvent { + expected_turn_id: Some(accepted.expected_turn_id), + accepted_turn_id: Some(accepted.turn_id.clone()), + num_input_images: accepted.num_input_images, + result: TurnSteerResult::Accepted, + rejection_reason: None, + created_at, + }, + ); + Ok(accepted.turn_id) + } + Err(rejected) => { + self.services.analytics_events_client.track_turn_steer( + rejected.tracking, + CodexTurnSteerEvent { + expected_turn_id: rejected.expected_turn_id, + accepted_turn_id: None, + num_input_images: rejected.num_input_images, + result: TurnSteerResult::Rejected, + rejection_reason: Some(rejected.error.to_turn_steer_rejection_reason()), + created_at, + }, + ); + Err(rejected.error) + } } + } + + async fn try_append_input_to_active_turn( + &self, + input: Vec, + expected_turn_id: Option<&str>, + ) -> Result { + let thread_id = self.conversation_id.to_string(); + let fallback_tracking = || { + build_track_events_context( + String::new(), + thread_id.clone(), + expected_turn_id.unwrap_or_default().to_string(), + ) + }; + + if input.is_empty() { + return Err(RejectedSteerInput { + error: SteerInputError::EmptyInput, + tracking: fallback_tracking(), + expected_turn_id: expected_turn_id.map(str::to_string), + num_input_images: 0, + }); + } + + let num_input_images = input + .iter() + .filter(|item| matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })) + .count(); let mut active = self.active_turn.lock().await; let Some(active_turn) = active.as_mut() else { - return Err(SteerInputError::NoActiveTurn(input)); + return Err(RejectedSteerInput { + error: SteerInputError::NoActiveTurn(input), + tracking: fallback_tracking(), + expected_turn_id: expected_turn_id.map(str::to_string), + num_input_images, + }); }; - let Some((active_turn_id, _)) = active_turn.tasks.first() else { - return Err(SteerInputError::NoActiveTurn(input)); + let Some((active_turn_id, task)) = active_turn.tasks.first() else { + return Err(RejectedSteerInput { + error: SteerInputError::NoActiveTurn(input), + tracking: fallback_tracking(), + expected_turn_id: expected_turn_id.map(str::to_string), + num_input_images, + }); }; + let active_turn_id = active_turn_id.clone(); + let tracking = build_track_events_context( + task.turn_context.model_info.slug.clone(), + thread_id.clone(), + task.turn_context.sub_id.clone(), + ); if let Some(expected_turn_id) = expected_turn_id && expected_turn_id != active_turn_id { - return Err(SteerInputError::ExpectedTurnMismatch { - expected: expected_turn_id.to_string(), - actual: active_turn_id.clone(), + return Err(RejectedSteerInput { + error: SteerInputError::ExpectedTurnMismatch { + expected: expected_turn_id.to_string(), + actual: active_turn_id, + }, + tracking, + expected_turn_id: Some(expected_turn_id.to_string()), + num_input_images, }); } match active_turn.tasks.first().map(|(_, task)| task.kind) { Some(crate::state::TaskKind::Regular) => {} Some(crate::state::TaskKind::Review) => { - return Err(SteerInputError::ActiveTurnNotSteerable { - turn_kind: NonSteerableTurnKind::Review, + return Err(RejectedSteerInput { + error: SteerInputError::ActiveTurnNotSteerable { + turn_kind: NonSteerableTurnKind::Review, + }, + tracking, + expected_turn_id: expected_turn_id + .map(str::to_string) + .or(Some(active_turn_id)), + num_input_images, }); } Some(crate::state::TaskKind::Compact) => { - return Err(SteerInputError::ActiveTurnNotSteerable { - turn_kind: NonSteerableTurnKind::Compact, + return Err(RejectedSteerInput { + error: SteerInputError::ActiveTurnNotSteerable { + turn_kind: NonSteerableTurnKind::Compact, + }, + tracking, + expected_turn_id: expected_turn_id + .map(str::to_string) + .or(Some(active_turn_id)), + num_input_images, + }); + } + None => { + return Err(RejectedSteerInput { + error: SteerInputError::NoActiveTurn(input), + tracking: fallback_tracking(), + expected_turn_id: expected_turn_id.map(str::to_string), + num_input_images, }); } - None => return Err(SteerInputError::NoActiveTurn(input)), } let mut turn_state = active_turn.turn_state.lock().await; turn_state.push_pending_input(input.into()); turn_state.accept_mailbox_delivery_for_current_turn(); - Ok(active_turn_id.clone()) + drop(turn_state); + + let expected_turn_id = expected_turn_id + .map(str::to_string) + .unwrap_or_else(|| active_turn_id.clone()); + + Ok(AcceptedSteerInput { + turn_id: active_turn_id, + tracking, + expected_turn_id, + num_input_images, + }) } /// Returns the input if there was no task running to inject into. @@ -4801,6 +4947,7 @@ fn submission_dispatch_span(sub: &Submission) -> tracing::Span { /// Operation handlers mod handlers { + use crate::codex::RejectedSteerInput; use crate::codex::Session; use crate::codex::SessionSettingsUpdate; use crate::codex::SteerInputError; @@ -4961,11 +5108,14 @@ mod handlers { sess.maybe_emit_unknown_model_warning_for_turn(current_context.as_ref()) .await; match sess - .steer_input(items.clone(), /*expected_turn_id*/ None) + .try_append_input_to_active_turn(items.clone(), /*expected_turn_id*/ None) .await { Ok(_) => current_context.session_telemetry.user_prompt(&items), - Err(SteerInputError::NoActiveTurn(items)) => { + Err(RejectedSteerInput { + error: SteerInputError::NoActiveTurn(items), + .. + }) => { current_context.session_telemetry.user_prompt(&items); sess.refresh_mcp_servers_if_requested(¤t_context) .await; @@ -4976,10 +5126,10 @@ mod handlers { ) .await; } - Err(err) => { + Err(RejectedSteerInput { error, .. }) => { sess.send_event_raw(Event { id: sub_id, - msg: EventMsg::Error(err.to_error_event()), + msg: EventMsg::Error(error.to_error_event()), }) .await; } From 075607294459b41d1dc272f4d0886e66c0ae1f07 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Tue, 7 Apr 2026 12:20:29 -0700 Subject: [PATCH 5/5] [codex-analytics] denormalize thread metadata onto turn events --- .../analytics/src/analytics_client_tests.rs | 249 ++++++++++++++---- codex-rs/analytics/src/events.rs | 49 +++- codex-rs/analytics/src/facts.rs | 12 + codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 38 ++- .../app-server/src/codex_message_processor.rs | 1 + .../app-server/tests/suite/v2/turn_start.rs | 13 +- codex-rs/core/src/codex.rs | 21 ++ codex-rs/core/src/codex_tests.rs | 7 + codex-rs/core/src/codex_thread.rs | 2 + 10 files changed, 315 insertions(+), 78 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index ffa1aa489c..d087518ab9 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -8,7 +8,6 @@ use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; use crate::events::CodexTurnEventRequest; use crate::events::CodexTurnSteerEventRequest; -use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; @@ -30,6 +29,7 @@ use crate::facts::PluginUsedInput; use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; @@ -73,6 +73,7 @@ use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::ModeKind; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::TokenUsage; use pretty_assertions::assert_eq; @@ -234,6 +235,9 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { thread_id: "thread-2".to_string(), num_input_images: 1, submission_type: Some(TurnSubmissionType::Default), + ephemeral: false, + session_source: SessionSource::Exec, + initialization_mode: ThreadInitializationMode::New, model: "gpt-5".to_string(), model_provider: "openai".to_string(), sandbox_policy: SandboxPolicy::new_read_only_policy(), @@ -252,8 +256,8 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { fn sample_app_server_client_metadata() -> CodexAppServerClientMetadata { CodexAppServerClientMetadata { product_client_id: "codex-tui".to_string(), - client_name: Some("codex-tui".to_string()), - client_version: Some("1.0.0".to_string()), + client_name: "codex-tui".to_string(), + client_version: "1.0.0".to_string(), rpc_transport: AppServerRpcTransport::Stdio, experimental_api_enabled: None, } @@ -535,8 +539,8 @@ fn thread_initialized_event_serializes_expected_shape() { thread_id: "thread-0".to_string(), app_server_client: CodexAppServerClientMetadata { product_client_id: DEFAULT_ORIGINATOR.to_string(), - client_name: Some("codex-tui".to_string()), - client_version: Some("1.0.0".to_string()), + client_name: "codex-tui".to_string(), + client_version: "1.0.0".to_string(), rpc_transport: AppServerRpcTransport::Stdio, experimental_api_enabled: Some(true), }, @@ -1073,8 +1077,14 @@ fn turn_event_serializes_expected_shape() { event_params: crate::events::CodexTurnEventParams { thread_id: "thread-2".to_string(), turn_id: "turn-2".to_string(), - product_client_id: "codex-tui".to_string(), + app_server_client: sample_app_server_client_metadata(), + runtime: sample_runtime_metadata(), submission_type: Some(TurnSubmissionType::Default), + ephemeral: false, + thread_source: Some("user".to_string()), + initialization_mode: ThreadInitializationMode::New, + subagent_source: None, + parent_thread_id: None, model: Some("gpt-5".to_string()), model_provider: "openai".to_string(), sandbox_policy: Some("read_only"), @@ -1111,51 +1121,97 @@ fn turn_event_serializes_expected_shape() { })); let payload = serde_json::to_value(&event).expect("serialize turn event"); - + assert_eq!(payload["event_type"], json!("codex_turn_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); assert_eq!( - payload, + payload["event_params"]["app_server_client"], json!({ - "event_type": "codex_turn_event", - "event_params": { - "thread_id": "thread-2", - "turn_id": "turn-2", - "product_client_id": "codex-tui", - "submission_type": "default", - "model": "gpt-5", - "model_provider": "openai", - "sandbox_policy": "read_only", - "reasoning_effort": "high", - "reasoning_summary": "detailed", - "service_tier": "flex", - "approval_policy": "on-request", - "approvals_reviewer": "guardian_subagent", - "sandbox_network_access": true, - "collaboration_mode": "plan", - "personality": "pragmatic", - "num_input_images": 2, - "is_first_turn": true, - "status": "completed", - "turn_error": null, - "steer_count": 0, - "total_tool_call_count": null, - "shell_command_count": null, - "file_change_count": null, - "mcp_tool_call_count": null, - "dynamic_tool_call_count": null, - "subagent_tool_call_count": null, - "web_search_count": null, - "image_generation_count": null, - "input_tokens": null, - "cached_input_tokens": null, - "output_tokens": null, - "reasoning_output_tokens": null, - "total_tokens": null, - "duration_ms": 1234, - "started_at": 455, - "completed_at": 456 - } + "product_client_id": "codex-tui", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": null, }) ); + assert_eq!( + payload["event_params"]["runtime"], + json!({ + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64", + }) + ); + assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["submission_type"], json!("default")); + assert_eq!(payload["event_params"]["ephemeral"], json!(false)); + assert_eq!(payload["event_params"]["thread_source"], json!("user")); + assert_eq!(payload["event_params"]["initialization_mode"], json!("new")); + assert_eq!(payload["event_params"]["subagent_source"], json!(null)); + assert_eq!(payload["event_params"]["parent_thread_id"], json!(null)); + assert_eq!(payload["event_params"]["model"], json!("gpt-5")); + assert_eq!(payload["event_params"]["model_provider"], json!("openai")); + assert_eq!( + payload["event_params"]["sandbox_policy"], + json!("read_only") + ); + assert_eq!(payload["event_params"]["reasoning_effort"], json!("high")); + assert_eq!( + payload["event_params"]["reasoning_summary"], + json!("detailed") + ); + assert_eq!(payload["event_params"]["service_tier"], json!("flex")); + assert_eq!( + payload["event_params"]["approval_policy"], + json!("on-request") + ); + assert_eq!( + payload["event_params"]["approvals_reviewer"], + json!("guardian_subagent") + ); + assert_eq!( + payload["event_params"]["sandbox_network_access"], + json!(true) + ); + assert_eq!(payload["event_params"]["collaboration_mode"], json!("plan")); + assert_eq!(payload["event_params"]["personality"], json!("pragmatic")); + assert_eq!(payload["event_params"]["num_input_images"], json!(2)); + assert_eq!(payload["event_params"]["is_first_turn"], json!(true)); + assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["turn_error"], json!(null)); + assert_eq!(payload["event_params"]["steer_count"], json!(0)); + assert_eq!( + payload["event_params"]["total_tool_call_count"], + json!(null) + ); + assert_eq!(payload["event_params"]["shell_command_count"], json!(null)); + assert_eq!(payload["event_params"]["file_change_count"], json!(null)); + assert_eq!(payload["event_params"]["mcp_tool_call_count"], json!(null)); + assert_eq!( + payload["event_params"]["dynamic_tool_call_count"], + json!(null) + ); + assert_eq!( + payload["event_params"]["subagent_tool_call_count"], + json!(null) + ); + assert_eq!(payload["event_params"]["web_search_count"], json!(null)); + assert_eq!( + payload["event_params"]["image_generation_count"], + json!(null) + ); + assert_eq!(payload["event_params"]["input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["output_tokens"], json!(null)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(null) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(null)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["started_at"], json!(455)); + assert_eq!(payload["event_params"]["completed_at"], json!(456)); } #[test] @@ -1464,9 +1520,30 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); assert_eq!( - payload["event_params"]["product_client_id"], - json!("codex-tui") + payload["event_params"]["app_server_client"], + json!({ + "product_client_id": "codex-tui", + "client_name": "codex-tui", + "client_version": "1.0.0", + "rpc_transport": "stdio", + "experimental_api_enabled": null, + }) ); + assert_eq!( + payload["event_params"]["runtime"], + json!({ + "codex_rs_version": "0.1.0", + "runtime_os": "macos", + "runtime_os_version": "15.3.1", + "runtime_arch": "aarch64", + }) + ); + assert!(payload["event_params"].get("product_client_id").is_none()); + assert_eq!(payload["event_params"]["ephemeral"], json!(false)); + assert_eq!(payload["event_params"]["thread_source"], json!("user")); + assert_eq!(payload["event_params"]["initialization_mode"], json!("new")); + assert_eq!(payload["event_params"]["subagent_source"], json!(null)); + assert_eq!(payload["event_params"]["parent_thread_id"], json!(null)); assert_eq!(payload["event_params"]["num_input_images"], json!(1)); assert_eq!(payload["event_params"]["status"], json!("completed")); assert_eq!(payload["event_params"]["steer_count"], json!(0)); @@ -1664,6 +1741,73 @@ async fn queued_submission_type_emits_queued_turn_event() { assert_eq!(payload["event_params"]["submission_type"], json!("queued")); } +#[tokio::test] +async fn turn_event_includes_subagent_thread_metadata() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + + let mut resolved_config = sample_turn_resolved_config("turn-2"); + resolved_config.ephemeral = true; + let parent_thread_id = + codex_protocol::ThreadId::from_string("11111111-1111-1111-1111-111111111111") + .expect("valid thread id"); + resolved_config.session_source = SessionSource::SubAgent(SubAgentSource::ThreadSpawn { + parent_thread_id, + depth: 1, + agent_path: None, + agent_nickname: Some("worker".to_string()), + agent_role: None, + }); + resolved_config.initialization_mode = ThreadInitializationMode::Forked; + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + resolved_config, + ))), + &mut out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["ephemeral"], json!(true)); + assert_eq!(payload["event_params"]["thread_source"], json!("subagent")); + assert_eq!( + payload["event_params"]["initialization_mode"], + json!("forked") + ); + assert_eq!( + payload["event_params"]["subagent_source"], + json!("thread_spawn") + ); + assert_eq!( + payload["event_params"]["parent_thread_id"], + json!("11111111-1111-1111-1111-111111111111") + ); +} + #[tokio::test] async fn turn_does_not_emit_without_required_prerequisites() { let mut reducer = AnalyticsReducer::default(); @@ -1689,12 +1833,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { &mut out, ) .await; - assert_eq!(out.len(), 1); - let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); - assert_eq!( - payload["event_params"]["product_client_id"], - json!(originator().value) - ); + assert!(out.is_empty()); let mut reducer = AnalyticsReducer::default(); let mut out = Vec::new(); diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 23e6cccbe4..e1c854f9ad 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -3,6 +3,7 @@ use crate::facts::CodexTurnSteerEvent; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnStatus; use crate::facts::TurnSteerRejectionReason; @@ -23,14 +24,6 @@ pub enum AppServerRpcTransport { InProcess, } -#[derive(Clone, Copy, Debug, Serialize)] -#[serde(rename_all = "snake_case")] -pub(crate) enum ThreadInitializationMode { - New, - Forked, - Resumed, -} - #[derive(Serialize)] pub(crate) struct TrackEventsRequest { pub(crate) events: Vec, @@ -73,8 +66,8 @@ pub(crate) struct SkillInvocationEventParams { #[derive(Clone, Serialize)] pub(crate) struct CodexAppServerClientMetadata { pub(crate) product_client_id: String, - pub(crate) client_name: Option, - pub(crate) client_version: Option, + pub(crate) client_name: String, + pub(crate) client_version: String, pub(crate) rpc_transport: AppServerRpcTransport, pub(crate) experimental_api_enabled: Option, } @@ -134,8 +127,14 @@ pub(crate) struct CodexAppUsedEventRequest { pub(crate) struct CodexTurnEventParams { pub(crate) thread_id: String, pub(crate) turn_id: String, - pub(crate) product_client_id: String, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, pub(crate) submission_type: Option, + pub(crate) ephemeral: bool, + pub(crate) thread_source: Option, + pub(crate) initialization_mode: ThreadInitializationMode, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, pub(crate) model: Option, pub(crate) model_provider: String, pub(crate) sandbox_policy: Option<&'static str>, @@ -330,8 +329,8 @@ pub(crate) fn subagent_thread_started_event_request( thread_id: input.thread_id, app_server_client: CodexAppServerClientMetadata { product_client_id: input.product_client_id, - client_name: Some(input.client_name), - client_version: Some(input.client_version), + client_name: input.client_name, + client_version: input.client_version, rpc_transport: AppServerRpcTransport::InProcess, experimental_api_enabled: None, }, @@ -368,3 +367,27 @@ fn subagent_parent_thread_id(subagent_source: &SubAgentSource) -> Option _ => None, } } + +pub(crate) fn turn_subagent_source_name(thread_source: &SessionSource) -> Option { + match thread_source { + SessionSource::SubAgent(subagent_source) => Some(subagent_source_name(subagent_source)), + SessionSource::Cli + | SessionSource::VSCode + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::Unknown => None, + } +} + +pub(crate) fn turn_parent_thread_id(thread_source: &SessionSource) -> Option { + match thread_source { + SessionSource::SubAgent(subagent_source) => subagent_parent_thread_id(subagent_source), + SessionSource::Cli + | SessionSource::VSCode + | SessionSource::Exec + | SessionSource::Mcp + | SessionSource::Custom(_) + | SessionSource::Unknown => None, + } +} diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 6c11b06c28..9b719cb3d4 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -14,6 +14,7 @@ use codex_protocol::config_types::ServiceTier; use codex_protocol::openai_models::ReasoningEffort; use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; use codex_protocol::protocol::TokenUsage; @@ -45,6 +46,9 @@ pub struct TurnResolvedConfigFact { pub thread_id: String, pub num_input_images: usize, pub submission_type: Option, + pub ephemeral: bool, + pub session_source: SessionSource, + pub initialization_mode: ThreadInitializationMode, pub model: String, pub model_provider: String, pub sandbox_policy: SandboxPolicy, @@ -73,6 +77,14 @@ pub enum TurnSubmissionType { Queued, } +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ThreadInitializationMode { + New, + Forked, + Resumed, +} + #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] pub enum TurnStatus { diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index b3e829ea03..8fa4da8228 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -10,6 +10,7 @@ pub use facts::CodexTurnSteerEvent; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::SubAgentThreadStartedInput; +pub use facts::ThreadInitializationMode; pub use facts::TrackEventsContext; pub use facts::TurnResolvedConfigFact; pub use facts::TurnStatus; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index d6f3ebd5e5..eead6f0309 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -10,7 +10,6 @@ use crate::events::CodexTurnEventRequest; use crate::events::CodexTurnSteerEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; -use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; use crate::events::TrackEventRequest; @@ -21,6 +20,8 @@ use crate::events::codex_turn_steer_event_params; use crate::events::plugin_state_event_type; use crate::events::subagent_thread_started_event_request; use crate::events::thread_source_name; +use crate::events::turn_parent_thread_id; +use crate::events::turn_subagent_source_name; use crate::facts::AnalyticsFact; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; @@ -31,6 +32,7 @@ use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; use crate::facts::SubAgentThreadStartedInput; +use crate::facts::ThreadInitializationMode; use crate::facts::TrackEventsContext; use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; @@ -178,8 +180,8 @@ impl AnalyticsReducer { ConnectionState { app_server_client: CodexAppServerClientMetadata { product_client_id, - client_name: Some(params.client_info.name), - client_version: Some(params.client_info.version), + client_name: params.client_info.name, + client_version: params.client_info.version, rpc_transport, experimental_api_enabled: params .capabilities @@ -584,16 +586,24 @@ impl AnalyticsReducer { { return; } - let product_client_id = turn_state + let connection_metadata = turn_state .connection_id .and_then(|connection_id| self.connections.get(&connection_id)) - .map(|connection_state| connection_state.app_server_client.product_client_id.clone()) - .unwrap_or_else(|| originator().value); + .map(|connection_state| { + ( + connection_state.app_server_client.clone(), + connection_state.runtime.clone(), + ) + }); + let Some((app_server_client, runtime)) = connection_metadata else { + return; + }; out.push(TrackEventRequest::TurnEvent(Box::new( CodexTurnEventRequest { event_type: "codex_turn_event", event_params: codex_turn_event_params( - product_client_id, + app_server_client, + runtime, turn_id.to_string(), turn_state, ), @@ -604,7 +614,8 @@ impl AnalyticsReducer { } fn codex_turn_event_params( - product_client_id: String, + app_server_client: CodexAppServerClientMetadata, + runtime: CodexRuntimeMetadata, turn_id: String, turn_state: &TurnState, ) -> CodexTurnEventParams { @@ -622,6 +633,9 @@ fn codex_turn_event_params( thread_id: _resolved_thread_id, num_input_images: _resolved_num_input_images, submission_type, + ephemeral, + session_source, + initialization_mode, model, model_provider, sandbox_policy, @@ -639,8 +653,14 @@ fn codex_turn_event_params( CodexTurnEventParams { thread_id, turn_id, - product_client_id, + app_server_client, + runtime, submission_type, + ephemeral, + thread_source: thread_source_name(&session_source).map(str::to_string), + initialization_mode, + subagent_source: turn_subagent_source_name(&session_source), + parent_thread_id: turn_parent_thread_id(&session_source), model: Some(model), model_provider, sandbox_policy: Some(sandbox_policy_mode(&sandbox_policy)), diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index 636be2e981..389b71cc39 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -9167,6 +9167,7 @@ mod tests { reasoning_effort: None, personality: None, session_source: SessionSource::Cli, + initialization_mode: codex_analytics::ThreadInitializationMode::New, }; assert_eq!( diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index a503c3087c..b6f6d60f5e 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -398,13 +398,24 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert_eq!(event["event_params"]["thread_id"], thread.id); assert_eq!(event["event_params"]["turn_id"], turn.id); assert_eq!( - event["event_params"]["product_client_id"], + event["event_params"]["app_server_client"]["product_client_id"], DEFAULT_CLIENT_NAME ); assert_eq!(event["event_params"]["model"], "mock-model"); assert_eq!(event["event_params"]["model_provider"], "mock_provider"); assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); assert_eq!(event["event_params"]["submission_type"], "default"); + assert_eq!(event["event_params"]["ephemeral"], false); + assert_eq!(event["event_params"]["thread_source"], "user"); + assert_eq!(event["event_params"]["initialization_mode"], "new"); + assert_eq!( + event["event_params"]["subagent_source"], + serde_json::Value::Null + ); + assert_eq!( + event["event_params"]["parent_thread_id"], + serde_json::Value::Null + ); assert_eq!(event["event_params"]["num_input_images"], 1); assert_eq!(event["event_params"]["is_first_turn"], true); assert_eq!(event["event_params"]["status"], "completed"); diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 81bc3d7141..776cdadb9d 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -52,6 +52,7 @@ use codex_analytics::AppInvocation; use codex_analytics::CodexTurnSteerEvent; use codex_analytics::InvocationType; use codex_analytics::SubAgentThreadStartedInput; +use codex_analytics::ThreadInitializationMode; use codex_analytics::TrackEventsContext; use codex_analytics::TurnResolvedConfigFact; use codex_analytics::TurnSteerRejectionReason; @@ -688,6 +689,7 @@ impl Codex { app_server_client_name: None, app_server_client_version: None, session_source, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools, persist_extended_history, inherited_shell_snapshot, @@ -1127,6 +1129,14 @@ fn turn_submission_type(submission_type: SubmissionType) -> TurnSubmissionType { } } +fn thread_initialization_mode(initial_history: &InitialHistory) -> ThreadInitializationMode { + match initial_history { + InitialHistory::New => ThreadInitializationMode::New, + InitialHistory::Forked(_) => ThreadInitializationMode::Forked, + InitialHistory::Resumed(_) => ThreadInitializationMode::Resumed, + } +} + fn local_time_context() -> (String, String) { match iana_time_zone::get_timezone() { Ok(timezone) => (Local::now().format("%Y-%m-%d").to_string(), timezone), @@ -1188,6 +1198,7 @@ pub(crate) struct SessionConfiguration { app_server_client_version: Option, /// Source of the session (cli, vscode, exec, mcp, ...) session_source: SessionSource, + thread_initialization_mode: ThreadInitializationMode, dynamic_tools: Vec, persist_extended_history: bool, inherited_shell_snapshot: Option>, @@ -1212,6 +1223,7 @@ impl SessionConfiguration { reasoning_effort: self.collaboration_mode.reasoning_effort(), personality: self.personality, session_source: self.session_source.clone(), + initialization_mode: self.thread_initialization_mode, } } @@ -1588,6 +1600,8 @@ impl Session { session_configuration.collaboration_mode.model(), session_configuration.provider ); + session_configuration.thread_initialization_mode = + thread_initialization_mode(&initial_history); let forked_from_id = initial_history.forked_from_id(); let (conversation_id, rollout_params) = match &initial_history { @@ -6528,6 +6542,10 @@ async fn track_turn_resolved_config_analytics( return; } + let thread_config = { + let state = sess.state.lock().await; + state.session_configuration.thread_config_snapshot() + }; let is_first_turn = { let mut state = sess.state.lock().await; state.take_next_turn_is_first() @@ -6549,6 +6567,9 @@ async fn track_turn_resolved_config_analytics( .map(turn_submission_type) .unwrap_or(TurnSubmissionType::Default), ), + ephemeral: thread_config.ephemeral, + session_source: thread_config.session_source, + initialization_mode: thread_config.initialization_mode, model: turn_context.model_info.slug.clone(), model_provider: turn_context.config.model_provider_id.clone(), sandbox_policy: turn_context.sandbox_policy.get().clone(), diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 055b83b952..5c828db59a 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -13,6 +13,7 @@ use crate::function_tool::FunctionCallError; use crate::shell::default_user_shell; use crate::tools::format_exec_output_str; +use codex_analytics::ThreadInitializationMode; use codex_features::Features; use codex_login::CodexAuth; use codex_mcp::mcp_connection_manager::ToolInfo; @@ -1875,6 +1876,7 @@ async fn set_rate_limits_retains_previous_credits() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools: Vec::new(), persist_extended_history: false, inherited_shell_snapshot: None, @@ -1977,6 +1979,7 @@ async fn set_rate_limits_updates_plan_type_when_present() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools: Vec::new(), persist_extended_history: false, inherited_shell_snapshot: None, @@ -2326,6 +2329,7 @@ pub(crate) async fn make_session_configuration_for_tests() -> SessionConfigurati app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools: Vec::new(), persist_extended_history: false, inherited_shell_snapshot: None, @@ -2592,6 +2596,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_zsh_path() { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools: Vec::new(), persist_extended_history: false, inherited_shell_snapshot: None, @@ -2696,6 +2701,7 @@ pub(crate) async fn make_session_and_context() -> (Session, TurnContext) { app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools: Vec::new(), persist_extended_history: false, inherited_shell_snapshot: None, @@ -3538,6 +3544,7 @@ pub(crate) async fn make_session_and_context_with_dynamic_tools_and_rx( app_server_client_name: None, app_server_client_version: None, session_source: SessionSource::Exec, + thread_initialization_mode: ThreadInitializationMode::New, dynamic_tools, persist_extended_history: false, inherited_shell_snapshot: None, diff --git a/codex-rs/core/src/codex_thread.rs b/codex-rs/core/src/codex_thread.rs index 9727cc208a..4a2f5dae24 100644 --- a/codex-rs/core/src/codex_thread.rs +++ b/codex-rs/core/src/codex_thread.rs @@ -3,6 +3,7 @@ use crate::codex::Codex; use crate::codex::SteerInputError; use crate::config::ConstraintResult; use crate::file_watcher::WatchRegistration; +use codex_analytics::ThreadInitializationMode; use codex_features::Feature; use codex_protocol::config_types::ApprovalsReviewer; use codex_protocol::config_types::Personality; @@ -42,6 +43,7 @@ pub struct ThreadConfigSnapshot { pub reasoning_effort: Option, pub personality: Option, pub session_source: SessionSource, + pub initialization_mode: ThreadInitializationMode, } pub struct CodexThread {