From 006911a31363d39d08849eec779949e345dd5f7c Mon Sep 17 00:00:00 2001 From: Chris Dong Date: Thu, 12 Nov 2026 23:42:18 +0000 Subject: [PATCH] Track dictation usage from accepted user inputs (#61215) ## What changed Capture `codex_app_used_dictation` as per-input metadata, with patches taking precedence over replacements, and remove the key from metadata forwarded to Responses. Record attribution when user-origin input is accepted into turn history. Add `used_dictation` to turn analytics: report `true` if any accepted input used dictation, `false` only when every accepted input has known non-dictation attribution, and omit the field otherwise. ## Testing Add coverage for per-input extraction, patch precedence, and turn-level aggregation. Extend the app-server steering test to verify that dictation attribution reaches analytics while the metadata key is excluded from Responses requests. GitOrigin-RevId: 0a55687a7dd61f00e9b188f89810e52bd941490b --- codex-rs/analytics/src/client.rs | 10 ++++ codex-rs/analytics/src/events.rs | 3 ++ codex-rs/analytics/src/facts.rs | 4 ++ codex-rs/analytics/src/reducer.rs | 26 ++++++++++ .../src/tests/suite/turns/events_tests.rs | 49 +++++++++++++++++++ .../tests/suite/v2/client_metadata.rs | 19 +++++-- codex-rs/core/src/session/input_queue.rs | 2 + codex-rs/core/src/session/mod.rs | 5 ++ codex-rs/core/src/session/turn_input.rs | 26 +++++++--- codex-rs/core/src/session/turn_input_tests.rs | 27 ++++++++++ 10 files changed, 159 insertions(+), 12 deletions(-) diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 6a9dd298ca..feb7728f19 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -745,6 +745,16 @@ impl AnalyticsEventsClient { ))); } + /// Record the provenance of one user input after it is added to turn history. + pub fn track_accepted_user_input(&self, turn_id: &str, used_dictation: Option) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::AcceptedUserInput { + turn_id: turn_id.to_owned(), + used_dictation, + }, + )); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index bb0a74d505..32fa452b8d 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -1174,6 +1174,9 @@ pub(crate) struct CodexTurnEventParams { /// The server-selected window responsible for a usage limit, when known. pub(crate) usage_limit_window_minutes: Option, pub(crate) steer_count: Option, + /// True when any accepted input used dictation. False requires known attribution for every input. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) used_dictation: Option, pub(crate) total_tool_call_count: Option, pub(crate) shell_command_count: Option, pub(crate) file_change_count: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 68d055c80f..dcebb56141 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -627,6 +627,10 @@ pub(crate) enum CustomAnalyticsFact { TurnTokenUsage(Box), TurnProfile(Box), TurnCodexError(Box), + AcceptedUserInput { + turn_id: String, + used_dictation: Option, + }, ImagePreparation(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index e9a59cac51..683685f71e 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -476,6 +476,14 @@ struct CompletedTurnState { duration_ms: Option, } +#[derive(Clone, Copy, Default)] +enum TurnDictation { + #[default] + NoUserInput, + Unknown, + Known(bool), +} + #[derive(Default)] struct TurnState { connection_id: Option, @@ -492,6 +500,7 @@ struct TurnState { codex_error: Option, latest_diff: Option, steer_count: usize, + dictation: TurnDictation, tool_counts: TurnToolCounts, completed_cua_tool_item_ids: HashSet, resource_skill_invocations: HashSet, @@ -817,6 +826,19 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnCodexError(input) => { self.ingest_turn_codex_error(*input); } + CustomAnalyticsFact::AcceptedUserInput { + turn_id, + used_dictation, + } => { + let state = self.turn_state(turn_id); + state.dictation = match (state.dictation, used_dictation) { + (TurnDictation::Known(true), _) | (_, Some(true)) => { + TurnDictation::Known(true) + } + (TurnDictation::Unknown, _) | (_, None) => TurnDictation::Unknown, + (_, Some(false)) => TurnDictation::Known(false), + }; + } CustomAnalyticsFact::ImagePreparation(input) => { self.ingest_image_preparation(*input); } @@ -4031,6 +4053,10 @@ fn codex_turn_event_params( codex_error_http_status_code: codex_error.and_then(|error| error.http_status_code), usage_limit_window_minutes: codex_error.and_then(|error| error.usage_limit_window_minutes), steer_count: Some(turn_state.steer_count), + used_dictation: match turn_state.dictation { + TurnDictation::Known(value) => Some(value), + TurnDictation::NoUserInput | TurnDictation::Unknown => None, + }, total_tool_call_count: Some(turn_state.tool_counts.total), shell_command_count: Some(turn_state.tool_counts.shell_command), file_change_count: Some(turn_state.tool_counts.file_change), diff --git a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs index 8f2bd1be7a..f9534d610b 100644 --- a/codex-rs/analytics/src/tests/suite/turns/events_tests.rs +++ b/codex-rs/analytics/src/tests/suite/turns/events_tests.rs @@ -154,6 +154,7 @@ fn turn_event_serializes_expected_shape() { codex_error_http_status_code: None, usage_limit_window_minutes: None, steer_count: Some(0), + used_dictation: None, total_tool_call_count: None, shell_command_count: None, file_change_count: None, @@ -789,6 +790,54 @@ async fn turn_does_not_emit_without_required_prerequisites() { assert!(out.is_empty()); } +#[tokio::test] +async fn turn_lifecycle_emits_dictation_from_accepted_inputs() { + for (inputs, expected) in [ + (&[Some(false)][..], Some(false)), + (&[Some(false), None][..], None), + (&[None, Some(true), Some(false)][..], Some(true)), + ] { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + /*include_token_usage*/ false, + ) + .await; + for &used_dictation in inputs { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::AcceptedUserInput { + turn_id: "turn-2".to_string(), + used_dictation, + }), + &mut out, + ) + .await; + } + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + let [event] = out.as_slice() else { + panic!("expected one turn event"); + }; + let payload = serde_json::to_value(event).expect("serialize turn event"); + assert_eq!(payload["event_params"]["used_dictation"], json!(expected)); + } +} + #[tokio::test] async fn turn_lifecycle_emits_failed_turn_event() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/app-server/tests/suite/v2/client_metadata.rs b/codex-rs/app-server/tests/suite/v2/client_metadata.rs index 76fe254834..d838befa69 100644 --- a/codex-rs/app-server/tests/suite/v2/client_metadata.rs +++ b/codex-rs/app-server/tests/suite/v2/client_metadata.rs @@ -26,6 +26,7 @@ use codex_protocol::protocol::SubAgentSource; use core_test_support::responses; use core_test_support::skip_if_no_network; use pretty_assertions::assert_eq; +use serde_json::json; use std::collections::HashMap; use tempfile::TempDir; use tokio::time::timeout; @@ -493,6 +494,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( let start_metadata = HashMap::from([ ("fiber_run_id".to_string(), "fiber-start-123".to_string()), + ("codex_app_used_dictation".to_string(), "false".to_string()), ("source".to_string(), "snapshot-source".to_string()), ("start_only".to_string(), "start-value".to_string()), ( @@ -530,6 +532,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( let steer_metadata = HashMap::from([ ("fiber_run_id".to_string(), "fiber-steer-456".to_string()), + ("codex_app_used_dictation".to_string(), "true".to_string()), ("origin".to_string(), "gaas".to_string()), ("source".to_string(), "steer-source".to_string()), ( @@ -579,6 +582,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( assert_eq!(first_metadata["turn_id"].as_str(), Some(turn_id.as_str())); assert_eq!(first_metadata["turn_trigger"].as_str(), Some("user")); assert_eq!(first_metadata["source"].as_str(), Some("initial-source")); + assert!(first_metadata.get("codex_app_used_dictation").is_none()); let second_metadata = requests[1] .header("x-codex-turn-metadata") @@ -593,6 +597,7 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( assert_eq!(second_metadata["turn_id"].as_str(), Some(turn_id.as_str())); assert_eq!(second_metadata["turn_trigger"].as_str(), Some("user")); assert_eq!(second_metadata["source"].as_str(), Some("steer-source")); + assert!(second_metadata.get("codex_app_used_dictation").is_none()); assert_eq!( second_metadata["start_only"].as_str(), patch_steer.then_some("start-value") @@ -603,12 +608,16 @@ async fn turn_steer_updates_client_metadata_on_follow_up_responses_request_v2( ); let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?; + let params = &event["event_params"]; assert_eq!( - ( - event["event_params"]["turn_trigger"].as_str(), - event["event_params"]["codex_turn_source"].as_str(), - ), - (Some("user"), Some("steer-source")) + json!([ + params["thread_id"], + params["turn_id"], + params["turn_trigger"], + params["codex_turn_source"], + params["used_dictation"], + ]), + json!([thread.id, turn_id, "user", "steer-source", true]), ); Ok(()) diff --git a/codex-rs/core/src/session/input_queue.rs b/codex-rs/core/src/session/input_queue.rs index f5aa2f69ef..a80ab9436f 100644 --- a/codex-rs/core/src/session/input_queue.rs +++ b/codex-rs/core/src/session/input_queue.rs @@ -30,6 +30,8 @@ pub struct UserInputMetadata { /// Optional host-assigned UUID for this particular Responses user item. #[serde(default, skip_serializing_if = "Option::is_none")] pub response_item_uuid: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_dictation: Option, #[serde( default, skip_serializing_if = "codex_history::UserInputOrigin::is_user" diff --git a/codex-rs/core/src/session/mod.rs b/codex-rs/core/src/session/mod.rs index 5d02f2b6d5..55e3c03c81 100644 --- a/codex-rs/core/src/session/mod.rs +++ b/codex-rs/core/src/session/mod.rs @@ -6003,6 +6003,11 @@ impl Session { image_preparations, ) .await; + if metadata.origin.is_user() { + self.services + .analytics_events_client + .track_accepted_user_input(&turn_context.sub_id, metadata.used_dictation); + } user_message_item.client_id = client_id; let turn_item = TurnItem::UserMessage(user_message_item); self.emit_turn_item_started(turn_context, &turn_item).await; diff --git a/codex-rs/core/src/session/turn_input.rs b/codex-rs/core/src/session/turn_input.rs index 3c9e91b6fa..d8c41b873d 100644 --- a/codex-rs/core/src/session/turn_input.rs +++ b/codex-rs/core/src/session/turn_input.rs @@ -380,8 +380,20 @@ pub(super) async fn handle_recovery( result } -fn user_input_metadata(request: &TurnInputRequest) -> UserInputMetadata { +fn user_input_metadata(request: &mut TurnInputRequest) -> UserInputMetadata { + const DICTATION_KEY: &str = "codex_app_used_dictation"; + // Capture provenance on this input before applying mutable turn metadata. + let metadata = &mut request.responsesapi_client_metadata; + let replacement = metadata + .replacement + .as_mut() + .and_then(|map| map.remove(DICTATION_KEY)); + let patch = metadata + .patch + .as_mut() + .and_then(|map| map.remove(DICTATION_KEY)); UserInputMetadata { + used_dictation: patch.or(replacement).and_then(|value| value.parse().ok()), origin: UserInputOrigin::from_turn_trigger(request.start.turn_trigger.as_deref()), response_item_uuid: request.user_item_uuid, extension_data: request.user_input_extension_data.clone(), @@ -401,10 +413,10 @@ async fn start_or_steer( return agent::submit(session, request, submission_id).await; } let WithTurnExtensionData { - request, + mut request, turn_extension_init, } = request; - let input_metadata = user_input_metadata(&request); + let input_metadata = user_input_metadata(&mut request); let TurnInputRequest { mut input, thread_settings, @@ -528,10 +540,10 @@ async fn start_if_idle( expected_previous_turn_id: Option, ) -> CodexResult { let WithTurnExtensionData { - request, + mut request, turn_extension_init, } = request; - let input_metadata = user_input_metadata(&request); + let input_metadata = user_input_metadata(&mut request); let TurnInputRequest { mut input, thread_settings, @@ -702,10 +714,10 @@ async fn steer( submission_id: String, ) -> CodexResult { let WithTurnExtensionData { - request, + mut request, turn_extension_init, } = request; - let input_metadata = user_input_metadata(&request); + let input_metadata = user_input_metadata(&mut request); let TurnInputRequest { mut input, thread_settings, diff --git a/codex-rs/core/src/session/turn_input_tests.rs b/codex-rs/core/src/session/turn_input_tests.rs index 2d49feb256..6dcd5a0457 100644 --- a/codex-rs/core/src/session/turn_input_tests.rs +++ b/codex-rs/core/src/session/turn_input_tests.rs @@ -1164,6 +1164,33 @@ async fn steer_preserves_request_origin( session.abort_all_tasks(TurnAbortReason::Interrupted).await; } +#[test] +fn dictation_metadata_is_captured_per_input_and_removed_from_responses() { + use std::collections::HashMap; + + let mut request = TurnInputRequest::user_input(Vec::new()) + .with_responses_metadata(Some(HashMap::from([ + ("source".to_owned(), "codex".to_owned()), + ("codex_app_used_dictation".to_owned(), "true".to_owned()), + ]))) + .with_responses_metadata_patch(Some(HashMap::from([( + "codex_app_used_dictation".to_owned(), + "false".to_owned(), + )]))); + assert_eq!( + user_input_metadata(&mut request).used_dictation, + Some(false) + ); + assert_eq!( + request.responsesapi_client_metadata.replacement, + Some(HashMap::from([("source".to_owned(), "codex".to_owned())])) + ); + assert_eq!( + request.responsesapi_client_metadata.patch, + Some(HashMap::new()) + ); +} + #[tokio::test] async fn scored_prompt_keeps_its_submission_item_id() { let (session, turn, events) = make_session_and_context_with_rx().await;