[codex-analytics] add steering metadata

This commit is contained in:
rhan-oai
2026-04-10 14:38:25 -07:00
parent 290c485ab5
commit b18388f2e6
9 changed files with 952 additions and 84 deletions

View File

@@ -16,6 +16,7 @@ use crate::events::codex_plugin_metadata;
use crate::events::codex_plugin_used_metadata;
use crate::events::subagent_thread_started_event_request;
use crate::facts::AnalyticsFact;
use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
@@ -27,6 +28,7 @@ use crate::facts::CompactionStatus;
use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::CustomAnalyticsFact;
use crate::facts::InputError;
use crate::facts::InvocationType;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
@@ -38,6 +40,7 @@ use crate::facts::ThreadInitializationMode;
use crate::facts::TrackEventsContext;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRequestError;
use crate::facts::TurnTokenUsageFact;
use crate::reducer::AnalyticsReducer;
use crate::reducer::normalize_path_for_skill_id;
@@ -47,8 +50,11 @@ use codex_app_server_protocol::AskForApproval as AppServerAskForApproval;
use codex_app_server_protocol::ClientInfo;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::InitializeCapabilities;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::NonSteerableTurnKind;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy;
use codex_app_server_protocol::ServerNotification;
@@ -63,6 +69,8 @@ use codex_app_server_protocol::TurnError as AppServerTurnError;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartedNotification;
use codex_app_server_protocol::TurnStatus as AppServerTurnStatus;
use codex_app_server_protocol::TurnSteerParams;
use codex_app_server_protocol::TurnSteerResponse;
use codex_app_server_protocol::UserInput;
use codex_login::default_client::DEFAULT_ORIGINATOR;
use codex_login::default_client::originator;
@@ -295,6 +303,160 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact {
}
}
fn sample_runtime_metadata() -> CodexRuntimeMetadata {
CodexRuntimeMetadata {
codex_rs_version: "0.1.0".to_string(),
runtime_os: "macos".to_string(),
runtime_os_version: "15.3.1".to_string(),
runtime_arch: "aarch64".to_string(),
}
}
fn sample_turn_steer_request(
thread_id: &str,
expected_turn_id: &str,
request_id: i64,
) -> ClientRequest {
ClientRequest::TurnSteer {
request_id: RequestId::Integer(request_id),
params: TurnSteerParams {
thread_id: thread_id.to_string(),
expected_turn_id: expected_turn_id.to_string(),
input: vec![
UserInput::Text {
text: "more".to_string(),
text_elements: vec![],
},
UserInput::LocalImage {
path: "/tmp/a.png".into(),
},
],
responsesapi_client_metadata: None,
},
}
}
fn sample_turn_steer_response(turn_id: &str, request_id: i64) -> ClientResponse {
ClientResponse::TurnSteer {
request_id: RequestId::Integer(request_id),
response: TurnSteerResponse {
turn_id: turn_id.to_string(),
},
}
}
fn no_active_turn_steer_error() -> JSONRPCErrorError {
JSONRPCErrorError {
code: -32600,
message: "no active turn to steer".to_string(),
data: None,
}
}
fn no_active_turn_steer_error_type() -> AnalyticsJsonRpcError {
AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::NoActiveTurn)
}
fn non_steerable_review_error() -> JSONRPCErrorError {
JSONRPCErrorError {
code: -32600,
message: "cannot steer a review turn".to_string(),
data: Some(
serde_json::to_value(AppServerTurnError {
message: "cannot steer a review turn".to_string(),
codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable {
turn_kind: NonSteerableTurnKind::Review,
}),
additional_details: None,
})
.expect("serialize turn error"),
),
}
}
fn non_steerable_review_error_type() -> AnalyticsJsonRpcError {
AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::ActiveTurnNotSteerable {
turn_kind: NonSteerableTurnKind::Review,
})
}
fn input_too_large_steer_error() -> JSONRPCErrorError {
JSONRPCErrorError {
code: -32602,
message: "Input exceeds the maximum length of 1048576 characters.".to_string(),
data: Some(json!({
"input_error_code": "input_too_large",
"actual_chars": 1048577,
"max_chars": 1048576,
})),
}
}
fn input_too_large_error_type() -> AnalyticsJsonRpcError {
AnalyticsJsonRpcError::Input(InputError::TooLarge)
}
async fn ingest_rejected_turn_steer(
reducer: &mut AnalyticsReducer,
out: &mut Vec<TrackEventRequest>,
error: JSONRPCErrorError,
error_type: Option<AnalyticsJsonRpcError>,
) -> serde_json::Value {
ingest_turn_prerequisites(
reducer, out, /*include_initialize*/ true, /*include_resolved_config*/ false,
/*include_started*/ false, /*include_token_usage*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(4),
request: Box::new(sample_turn_steer_request(
"thread-2", "turn-2", /*request_id*/ 4,
)),
},
out,
)
.await;
reducer
.ingest(
AnalyticsFact::ErrorResponse {
connection_id: 7,
request_id: RequestId::Integer(4),
error,
error_type,
},
out,
)
.await;
assert_eq!(out.len(), 1);
serde_json::to_value(&out[0]).expect("serialize turn steer event")
}
async fn ingest_initialize(reducer: &mut AnalyticsReducer, out: &mut Vec<TrackEventRequest>) {
reducer
.ingest(
AnalyticsFact::Initialize {
connection_id: 7,
params: InitializeParams {
client_info: ClientInfo {
name: "codex-tui".to_string(),
title: None,
version: "1.0.0".to_string(),
},
capabilities: None,
},
product_client_id: "codex-tui".to_string(),
runtime: sample_runtime_metadata(),
rpc_transport: AppServerRpcTransport::Stdio,
},
out,
)
.await;
}
async fn ingest_turn_prerequisites(
reducer: &mut AnalyticsReducer,
out: &mut Vec<TrackEventRequest>,
@@ -304,30 +466,7 @@ async fn ingest_turn_prerequisites(
include_token_usage: bool,
) {
if include_initialize {
reducer
.ingest(
AnalyticsFact::Initialize {
connection_id: 7,
params: InitializeParams {
client_info: ClientInfo {
name: "codex-tui".to_string(),
title: None,
version: "1.0.0".to_string(),
},
capabilities: None,
},
product_client_id: "codex-tui".to_string(),
runtime: CodexRuntimeMetadata {
codex_rs_version: "0.1.0".to_string(),
runtime_os: "macos".to_string(),
runtime_os_version: "15.3.1".to_string(),
runtime_arch: "aarch64".to_string(),
},
rpc_transport: AppServerRpcTransport::Stdio,
},
out,
)
.await;
ingest_initialize(reducer, out).await;
}
reducer
@@ -1338,7 +1477,7 @@ fn turn_event_serializes_expected_shape() {
is_first_turn: true,
status: Some(TurnStatus::Completed),
turn_error: None,
steer_count: None,
steer_count: Some(0),
total_tool_call_count: None,
shell_command_count: None,
file_change_count: None,
@@ -1399,7 +1538,7 @@ fn turn_event_serializes_expected_shape() {
"is_first_turn": true,
"status": "completed",
"turn_error": null,
"steer_count": null,
"steer_count": 0,
"total_tool_call_count": null,
"shell_command_count": null,
"file_change_count": null,
@@ -1424,6 +1563,226 @@ fn turn_event_serializes_expected_shape() {
assert_eq!(payload, expected);
}
#[tokio::test]
async fn accepted_turn_steer_emits_expected_event() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ false,
/*include_started*/ false,
/*include_token_usage*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(4),
request: Box::new(sample_turn_steer_request(
"thread-2", "turn-2", /*request_id*/ 4,
)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Response {
connection_id: 7,
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)),
},
&mut out,
)
.await;
assert_eq!(out.len(), 1);
let payload = serde_json::to_value(&out[0]).expect("serialize turn steer event");
assert_eq!(payload["event_type"], json!("codex_turn_steer_event"));
assert_eq!(payload["event_params"]["thread_id"], json!("thread-2"));
assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2"));
assert_eq!(payload["event_params"]["accepted_turn_id"], json!("turn-2"));
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
assert_eq!(payload["event_params"]["result"], json!("accepted"));
assert_eq!(payload["event_params"]["rejection_reason"], json!(null));
assert!(
payload["event_params"]["created_at"]
.as_u64()
.expect("created_at")
> 0
);
assert_eq!(
payload["event_params"]["app_server_client"]["product_client_id"],
json!("codex-tui")
);
assert_eq!(
payload["event_params"]["runtime"]["codex_rs_version"],
json!("0.1.0")
);
assert!(payload["event_params"].get("product_client_id").is_none());
}
#[tokio::test]
async fn rejected_turn_steer_uses_request_connection_metadata() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
let payload = ingest_rejected_turn_steer(
&mut reducer,
&mut out,
no_active_turn_steer_error(),
Some(no_active_turn_steer_error_type()),
)
.await;
assert_eq!(payload["event_type"], json!("codex_turn_steer_event"));
assert_eq!(payload["event_params"]["thread_id"], json!("thread-2"));
assert_eq!(payload["event_params"]["expected_turn_id"], json!("turn-2"));
assert_eq!(payload["event_params"]["accepted_turn_id"], json!(null));
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
assert_eq!(
payload["event_params"]["app_server_client"]["product_client_id"],
json!("codex-tui")
);
assert_eq!(
payload["event_params"]["runtime"]["codex_rs_version"],
json!("0.1.0")
);
assert_eq!(payload["event_params"]["result"], json!("rejected"));
assert_eq!(
payload["event_params"]["rejection_reason"],
json!("no_active_turn")
);
assert!(
payload["event_params"]["created_at"]
.as_u64()
.expect("created_at")
> 0
);
}
#[tokio::test]
async fn rejected_turn_steer_maps_active_turn_not_steerable_error_type() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
let payload = ingest_rejected_turn_steer(
&mut reducer,
&mut out,
non_steerable_review_error(),
Some(non_steerable_review_error_type()),
)
.await;
assert_eq!(
payload["event_params"]["rejection_reason"],
json!("non_steerable_review")
);
}
#[tokio::test]
async fn rejected_turn_steer_maps_input_too_large_error_type() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
let payload = ingest_rejected_turn_steer(
&mut reducer,
&mut out,
input_too_large_steer_error(),
Some(input_too_large_error_type()),
)
.await;
assert_eq!(
payload["event_params"]["rejection_reason"],
json!("input_too_large")
);
}
#[tokio::test]
async fn turn_steer_does_not_emit_without_pending_request() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
reducer
.ingest(
AnalyticsFact::ErrorResponse {
connection_id: 7,
request_id: RequestId::Integer(4),
error: no_active_turn_steer_error(),
error_type: Some(no_active_turn_steer_error_type()),
},
&mut out,
)
.await;
assert!(out.is_empty());
}
#[tokio::test]
async fn turn_start_error_response_discards_pending_start_request() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_initialize(&mut reducer, &mut out).await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(3),
request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::ErrorResponse {
connection_id: 7,
request_id: RequestId::Integer(3),
error: no_active_turn_steer_error(),
error_type: None,
},
&mut out,
)
.await;
// A late/synthetic response for the same request id must not resurrect the
// failed turn/start request and attach request-scoped connection metadata.
reducer
.ingest(
AnalyticsFact::Response {
connection_id: 7,
response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)),
},
&mut out,
)
.await;
assert!(out.is_empty());
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new(
sample_turn_resolved_config("turn-2"),
))),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
assert!(out.is_empty());
}
#[tokio::test]
async fn turn_lifecycle_emits_turn_event() {
let mut reducer = AnalyticsReducer::default();
@@ -1482,6 +1841,7 @@ async fn turn_lifecycle_emits_turn_event() {
assert_eq!(payload["event_params"]["parent_thread_id"], json!(null));
assert_eq!(payload["event_params"]["num_input_images"], json!(1));
assert_eq!(payload["event_params"]["status"], json!("completed"));
assert_eq!(payload["event_params"]["steer_count"], json!(0));
assert_eq!(payload["event_params"]["started_at"], json!(455));
assert_eq!(payload["event_params"]["completed_at"], json!(456));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
@@ -1495,6 +1855,109 @@ async fn turn_lifecycle_emits_turn_event() {
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
}
#[tokio::test]
async fn accepted_steers_increment_turn_steer_count() {
let mut reducer = AnalyticsReducer::default();
let mut out = Vec::new();
ingest_turn_prerequisites(
&mut reducer,
&mut out,
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ true,
/*include_token_usage*/ false,
)
.await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(4),
request: Box::new(sample_turn_steer_request(
"thread-2", "turn-2", /*request_id*/ 4,
)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Response {
connection_id: 7,
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 4)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(5),
request: Box::new(sample_turn_steer_request(
"thread-2", "turn-2", /*request_id*/ 5,
)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::ErrorResponse {
connection_id: 7,
request_id: RequestId::Integer(5),
error: no_active_turn_steer_error(),
error_type: Some(no_active_turn_steer_error_type()),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Request {
connection_id: 7,
request_id: RequestId::Integer(6),
request: Box::new(sample_turn_steer_request(
"thread-2", "turn-2", /*request_id*/ 6,
)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Response {
connection_id: 7,
response: Box::new(sample_turn_steer_response("turn-2", /*request_id*/ 6)),
},
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
"thread-2",
"turn-2",
AppServerTurnStatus::Completed,
/*codex_error_info*/ None,
))),
&mut out,
)
.await;
let turn_event = out
.iter()
.find(|event| matches!(event, TrackEventRequest::TurnEvent(_)))
.expect("turn event should be emitted");
let payload = serde_json::to_value(turn_event).expect("serialize turn event");
assert_eq!(payload["event_params"]["steer_count"], json!(2));
}
#[tokio::test]
async fn turn_does_not_emit_without_required_prerequisites() {
let mut reducer = AnalyticsReducer::default();

View File

@@ -3,6 +3,7 @@ use crate::events::TrackEventRequest;
use crate::events::TrackEventsRequest;
use crate::events::current_runtime_metadata;
use crate::facts::AnalyticsFact;
use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
@@ -19,6 +20,7 @@ use crate::reducer::AnalyticsReducer;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_login::AuthManager;
@@ -259,6 +261,21 @@ impl AnalyticsEventsClient {
});
}
pub fn track_error_response(
&self,
connection_id: u64,
request_id: RequestId,
error: JSONRPCErrorError,
error_type: Option<AnalyticsJsonRpcError>,
) {
self.record_fact(AnalyticsFact::ErrorResponse {
connection_id,
request_id,
error,
error_type,
});
}
pub fn track_notification(&self, notification: ServerNotification) {
self.record_fact(AnalyticsFact::Notification(Box::new(notification)));
}

View File

@@ -1,11 +1,14 @@
use crate::facts::AppInvocation;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexTurnSteerEvent;
use crate::facts::InvocationType;
use crate::facts::PluginState;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::ThreadInitializationMode;
use crate::facts::TrackEventsContext;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerResult;
use crate::facts::TurnSubmissionType;
use codex_app_server_protocol::CodexErrorInfo;
use codex_login::default_client::originator;
@@ -36,6 +39,7 @@ pub(crate) enum TrackEventRequest {
AppUsed(CodexAppUsedEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
PluginUsed(CodexPluginUsedEventRequest),
PluginInstalled(CodexPluginEventRequest),
PluginUninstalled(CodexPluginEventRequest),
@@ -202,6 +206,25 @@ pub(crate) struct CodexTurnEventRequest {
pub(crate) event_params: CodexTurnEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnSteerEventParams {
pub(crate) thread_id: String,
pub(crate) expected_turn_id: Option<String>,
pub(crate) accepted_turn_id: Option<String>,
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
pub(crate) num_input_images: usize,
pub(crate) result: TurnSteerResult,
pub(crate) rejection_reason: Option<TurnSteerRejectionReason>,
pub(crate) created_at: u64,
}
#[derive(Serialize)]
pub(crate) struct CodexTurnSteerEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: CodexTurnSteerEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexPluginMetadata {
pub(crate) plugin_id: Option<String>,
@@ -324,6 +347,25 @@ pub(crate) fn codex_plugin_used_metadata(
}
}
pub(crate) fn codex_turn_steer_event_params(
app_server_client: CodexAppServerClientMetadata,
runtime: CodexRuntimeMetadata,
tracking: &TrackEventsContext,
turn_steer: CodexTurnSteerEvent,
) -> CodexTurnSteerEventParams {
CodexTurnSteerEventParams {
thread_id: tracking.thread_id.clone(),
expected_turn_id: turn_steer.expected_turn_id,
accepted_turn_id: turn_steer.accepted_turn_id,
app_server_client,
runtime,
num_input_images: turn_steer.num_input_images,
result: turn_steer.result,
rejection_reason: turn_steer.rejection_reason,
created_at: turn_steer.created_at,
}
}
pub(crate) fn thread_source_name(thread_source: &SessionSource) -> Option<&'static str> {
match thread_source {
SessionSource::Cli | SessionSource::VSCode | SessionSource::Exec => Some("user"),

View File

@@ -3,6 +3,8 @@ use crate::events::CodexRuntimeMetadata;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::JSONRPCErrorError;
use codex_app_server_protocol::NonSteerableTurnKind;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_plugin::PluginTelemetryMetadata;
@@ -93,6 +95,53 @@ pub enum TurnStatus {
Interrupted,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnSteerResult {
Accepted,
Rejected,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnSteerRejectionReason {
NoActiveTurn,
ExpectedTurnMismatch,
NonSteerableReview,
NonSteerableCompact,
EmptyInput,
InputTooLarge,
}
#[derive(Clone)]
pub struct CodexTurnSteerEvent {
pub expected_turn_id: Option<String>,
pub accepted_turn_id: Option<String>,
pub num_input_images: usize,
pub result: TurnSteerResult,
pub rejection_reason: Option<TurnSteerRejectionReason>,
pub created_at: u64,
}
#[derive(Clone, Copy, Debug)]
pub enum AnalyticsJsonRpcError {
TurnSteer(TurnSteerRequestError),
Input(InputError),
}
#[derive(Clone, Copy, Debug)]
pub enum TurnSteerRequestError {
NoActiveTurn,
ExpectedTurnMismatch,
ActiveTurnNotSteerable { turn_kind: NonSteerableTurnKind },
}
#[derive(Clone, Copy, Debug)]
pub enum InputError {
Empty,
TooLarge,
}
#[derive(Clone, Debug)]
pub struct SkillInvocation {
pub skill_name: String,
@@ -208,6 +257,12 @@ pub(crate) enum AnalyticsFact {
connection_id: u64,
response: Box<ClientResponse>,
},
ErrorResponse {
connection_id: u64,
request_id: RequestId,
error: JSONRPCErrorError,
error_type: Option<AnalyticsJsonRpcError>,
},
Notification(Box<ServerNotification>),
// Facts that do not naturally exist on the app-server protocol surface, or
// would require non-trivial protocol reshaping on this branch.

View File

@@ -5,14 +5,17 @@ mod reducer;
pub use client::AnalyticsEventsClient;
pub use events::AppServerRpcTransport;
pub use facts::AnalyticsJsonRpcError;
pub use facts::AppInvocation;
pub use facts::CodexCompactionEvent;
pub use facts::CodexTurnSteerEvent;
pub use facts::CompactionImplementation;
pub use facts::CompactionPhase;
pub use facts::CompactionReason;
pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
pub use facts::InputError;
pub use facts::InvocationType;
pub use facts::SkillInvocation;
pub use facts::SubAgentThreadStartedInput;
@@ -20,6 +23,9 @@ pub use facts::ThreadInitializationMode;
pub use facts::TrackEventsContext;
pub use facts::TurnResolvedConfigFact;
pub use facts::TurnStatus;
pub use facts::TurnSteerRejectionReason;
pub use facts::TurnSteerRequestError;
pub use facts::TurnSteerResult;
pub use facts::TurnTokenUsageFact;
pub use facts::build_track_events_context;

View File

@@ -8,6 +8,7 @@ use crate::events::CodexPluginUsedEventRequest;
use crate::events::CodexRuntimeMetadata;
use crate::events::CodexTurnEventParams;
use crate::events::CodexTurnEventRequest;
use crate::events::CodexTurnSteerEventRequest;
use crate::events::SkillInvocationEventParams;
use crate::events::SkillInvocationEventRequest;
use crate::events::ThreadInitializedEvent;
@@ -17,31 +18,41 @@ use crate::events::codex_app_metadata;
use crate::events::codex_compaction_event_params;
use crate::events::codex_plugin_metadata;
use crate::events::codex_plugin_used_metadata;
use crate::events::codex_turn_steer_event_params;
use crate::events::plugin_state_event_type;
use crate::events::subagent_parent_thread_id;
use crate::events::subagent_source_name;
use crate::events::subagent_thread_started_event_request;
use crate::events::thread_source_name;
use crate::facts::AnalyticsFact;
use crate::facts::AnalyticsJsonRpcError;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexTurnSteerEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::InputError;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
use crate::facts::PluginUsedInput;
use crate::facts::SkillInvokedInput;
use crate::facts::SubAgentThreadStartedInput;
use crate::facts::ThreadInitializationMode;
use crate::facts::TrackEventsContext;
use crate::facts::TurnResolvedConfigFact;
use crate::facts::TurnStatus;
use crate::facts::TurnSteerRejectionReason;
use crate::facts::TurnSteerRequestError;
use crate::facts::TurnSteerResult;
use crate::facts::TurnTokenUsageFact;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::NonSteerableTurnKind;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::TurnSteerResponse;
use codex_app_server_protocol::UserInput;
use codex_git_utils::collect_git_info;
use codex_git_utils::get_git_repo_root;
@@ -56,6 +67,8 @@ use codex_protocol::protocol::TokenUsage;
use sha1::Digest;
use std::collections::HashMap;
use std::path::Path;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
#[derive(Default)]
pub(crate) struct AnalyticsReducer {
@@ -102,6 +115,7 @@ impl ThreadMetadataState {
enum RequestState {
TurnStart(PendingTurnStartState),
TurnSteer(PendingTurnSteerState),
}
struct PendingTurnStartState {
@@ -109,6 +123,13 @@ struct PendingTurnStartState {
num_input_images: usize,
}
struct PendingTurnSteerState {
thread_id: String,
expected_turn_id: String,
num_input_images: usize,
created_at: u64,
}
#[derive(Clone)]
struct CompletedTurnState {
status: Option<TurnStatus>,
@@ -125,6 +146,7 @@ struct TurnState {
started_at: Option<u64>,
token_usage: Option<TokenUsage>,
completed: Option<CompletedTurnState>,
steer_count: usize,
}
impl AnalyticsReducer {
@@ -158,6 +180,14 @@ impl AnalyticsReducer {
} => {
self.ingest_response(connection_id, *response, out);
}
AnalyticsFact::ErrorResponse {
connection_id,
request_id,
error: _,
error_type,
} => {
self.ingest_error_response(connection_id, request_id, error_type, out);
}
AnalyticsFact::Notification(notification) => {
self.ingest_notification(*notification, out);
}
@@ -234,22 +264,29 @@ impl AnalyticsReducer {
request_id: RequestId,
request: ClientRequest,
) {
let ClientRequest::TurnStart { params, .. } = request else {
return;
};
self.requests.insert(
(connection_id, request_id),
RequestState::TurnStart(PendingTurnStartState {
thread_id: params.thread_id,
num_input_images: params
.input
.iter()
.filter(|item| {
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
})
.count(),
}),
);
match request {
ClientRequest::TurnStart { params, .. } => {
self.requests.insert(
(connection_id, request_id),
RequestState::TurnStart(PendingTurnStartState {
thread_id: params.thread_id,
num_input_images: num_input_images(&params.input),
}),
);
}
ClientRequest::TurnSteer { params, .. } => {
self.requests.insert(
(connection_id, request_id),
RequestState::TurnSteer(PendingTurnSteerState {
thread_id: params.thread_id,
expected_turn_id: params.expected_turn_id,
num_input_images: num_input_images(&params.input),
created_at: now_unix_seconds(),
}),
);
}
_ => {}
}
}
fn ingest_turn_resolved_config(
@@ -268,6 +305,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.thread_id = Some(thread_id);
turn_state.num_input_images = Some(num_input_images);
@@ -289,6 +327,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.thread_id = Some(input.thread_id);
turn_state.token_usage = Some(input.token_usage);
@@ -441,16 +480,73 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.connection_id = Some(connection_id);
turn_state.thread_id = Some(pending_request.thread_id);
turn_state.num_input_images = Some(pending_request.num_input_images);
self.maybe_emit_turn_event(&turn_id, out);
}
ClientResponse::TurnSteer {
request_id,
response,
} => {
self.ingest_turn_steer_response(connection_id, request_id, response, out);
}
_ => {}
}
}
fn ingest_error_response(
&mut self,
connection_id: u64,
request_id: RequestId,
error_type: Option<AnalyticsJsonRpcError>,
out: &mut Vec<TrackEventRequest>,
) {
let Some(request) = self.requests.remove(&(connection_id, request_id)) else {
return;
};
self.ingest_request_error_response(connection_id, request, error_type, out);
}
fn ingest_request_error_response(
&mut self,
connection_id: u64,
request: RequestState,
error_type: Option<AnalyticsJsonRpcError>,
out: &mut Vec<TrackEventRequest>,
) {
match request {
RequestState::TurnStart(_) => {}
RequestState::TurnSteer(pending_request) => {
self.ingest_turn_steer_error_response(
connection_id,
pending_request,
error_type,
out,
);
}
}
}
fn ingest_turn_steer_error_response(
&mut self,
connection_id: u64,
pending_request: PendingTurnSteerState,
error_type: Option<AnalyticsJsonRpcError>,
out: &mut Vec<TrackEventRequest>,
) {
self.emit_turn_steer_event(
connection_id,
pending_request,
/*accepted_turn_id*/ None,
TurnSteerResult::Rejected,
rejection_reason_from_error_type(error_type),
out,
);
}
fn ingest_notification(
&mut self,
notification: ServerNotification,
@@ -466,6 +562,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.started_at = notification
.turn
@@ -484,6 +581,7 @@ impl AnalyticsReducer {
started_at: None,
token_usage: None,
completed: None,
steer_count: 0,
});
turn_state.completed = Some(CompletedTurnState {
status: analytics_turn_status(notification.turn.status),
@@ -586,6 +684,70 @@ impl AnalyticsReducer {
)));
}
fn ingest_turn_steer_response(
&mut self,
connection_id: u64,
request_id: RequestId,
response: TurnSteerResponse,
out: &mut Vec<TrackEventRequest>,
) {
let Some(RequestState::TurnSteer(pending_request)) =
self.requests.remove(&(connection_id, request_id))
else {
return;
};
if let Some(turn_state) = self.turns.get_mut(&response.turn_id) {
turn_state.steer_count += 1;
}
self.emit_turn_steer_event(
connection_id,
pending_request,
Some(response.turn_id),
TurnSteerResult::Accepted,
/*rejection_reason*/ None,
out,
);
}
fn emit_turn_steer_event(
&mut self,
connection_id: u64,
pending_request: PendingTurnSteerState,
accepted_turn_id: Option<String>,
result: TurnSteerResult,
rejection_reason: Option<TurnSteerRejectionReason>,
out: &mut Vec<TrackEventRequest>,
) {
let Some(connection_state) = self.connections.get(&connection_id) else {
return;
};
let tracking = TrackEventsContext {
model_slug: String::new(),
thread_id: pending_request.thread_id,
turn_id: accepted_turn_id
.as_deref()
.unwrap_or(pending_request.expected_turn_id.as_str())
.to_string(),
};
let turn_steer = CodexTurnSteerEvent {
expected_turn_id: Some(pending_request.expected_turn_id),
accepted_turn_id,
num_input_images: pending_request.num_input_images,
result,
rejection_reason,
created_at: pending_request.created_at,
};
out.push(TrackEventRequest::TurnSteer(CodexTurnSteerEventRequest {
event_type: "codex_turn_steer_event",
event_params: codex_turn_steer_event_params(
connection_state.app_server_client.clone(),
connection_state.runtime.clone(),
&tracking,
turn_steer,
),
}));
}
fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec<TrackEventRequest>) {
let Some(turn_state) = self.turns.get(turn_id) else {
return;
@@ -689,7 +851,7 @@ fn codex_turn_event_params(
is_first_turn,
status: completed.status,
turn_error: completed.turn_error,
steer_count: None,
steer_count: Some(turn_state.steer_count),
total_tool_call_count: None,
shell_command_count: None,
file_change_count: None,
@@ -758,6 +920,46 @@ fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Optio
}
}
fn num_input_images(input: &[UserInput]) -> usize {
input
.iter()
.filter(|item| matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }))
.count()
}
fn now_unix_seconds() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn rejection_reason_from_error_type(
error_type: Option<AnalyticsJsonRpcError>,
) -> Option<TurnSteerRejectionReason> {
match error_type {
Some(AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::NoActiveTurn)) => {
Some(TurnSteerRejectionReason::NoActiveTurn)
}
Some(AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::ExpectedTurnMismatch)) => {
Some(TurnSteerRejectionReason::ExpectedTurnMismatch)
}
Some(AnalyticsJsonRpcError::TurnSteer(TurnSteerRequestError::ActiveTurnNotSteerable {
turn_kind,
})) => match turn_kind {
NonSteerableTurnKind::Review => Some(TurnSteerRejectionReason::NonSteerableReview),
NonSteerableTurnKind::Compact => Some(TurnSteerRejectionReason::NonSteerableCompact),
},
Some(AnalyticsJsonRpcError::Input(InputError::Empty)) => {
Some(TurnSteerRejectionReason::EmptyInput)
}
Some(AnalyticsJsonRpcError::Input(InputError::TooLarge)) => {
Some(TurnSteerRejectionReason::InputTooLarge)
}
None => None,
}
}
pub(crate) fn skill_id_for_local_skill(
repo_url: Option<&str>,
repo_root: Option<&Path>,

View File

@@ -22,6 +22,9 @@ use chrono::DateTime;
use chrono::SecondsFormat;
use chrono::Utc;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::AnalyticsJsonRpcError;
use codex_analytics::InputError;
use codex_analytics::TurnSteerRequestError;
use codex_app_server_protocol::Account;
use codex_app_server_protocol::AccountLoginCompletedNotification;
use codex_app_server_protocol::AccountUpdatedNotification;
@@ -36,7 +39,7 @@ use codex_app_server_protocol::CancelLoginAccountResponse;
use codex_app_server_protocol::CancelLoginAccountStatus;
use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::ClientResponse;
use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::CollaborationModeListParams;
use codex_app_server_protocol::CollaborationModeListResponse;
use codex_app_server_protocol::CommandExecParams;
@@ -606,6 +609,22 @@ impl CodexMessageProcessor {
);
}
fn track_error_response(
&self,
request_id: &ConnectionRequestId,
error: &JSONRPCErrorError,
error_type: Option<AnalyticsJsonRpcError>,
) {
if self.config.features.enabled(Feature::GeneralAnalytics) {
self.analytics_events_client.track_error_response(
request_id.connection_id.0,
request_id.request_id.clone(),
error.clone(),
error_type,
);
}
}
async fn load_thread(
&self,
thread_id: &str,
@@ -6736,12 +6755,18 @@ impl CodexMessageProcessor {
app_server_client_version: Option<String>,
) {
if let Err(error) = Self::validate_v2_input_limit(&params.input) {
self.track_error_response(
&request_id,
&error,
Some(AnalyticsJsonRpcError::Input(InputError::TooLarge)),
);
self.outgoing.send_error(request_id, error).await;
return;
}
let (_, thread) = match self.load_thread(&params.thread_id).await {
Ok(v) => v,
Err(error) => {
self.track_error_response(&request_id, &error, /*error_type*/ None);
self.outgoing.send_error(request_id, error).await;
return;
}
@@ -6753,6 +6778,7 @@ impl CodexMessageProcessor {
)
.await
{
self.track_error_response(&request_id, &error, /*error_type*/ None);
self.outgoing.send_error(request_id, error).await;
return;
}
@@ -6853,6 +6879,7 @@ impl CodexMessageProcessor {
message: format!("failed to start turn: {err}"),
data: None,
};
self.track_error_response(&request_id, &error, /*error_type*/ None);
self.outgoing.send_error(request_id, error).await;
}
}
@@ -6877,23 +6904,31 @@ impl CodexMessageProcessor {
let (_, thread) = match self.load_thread(&params.thread_id).await {
Ok(v) => v,
Err(error) => {
self.track_error_response(&request_id, &error, /*error_type*/ None);
self.outgoing.send_error(request_id, error).await;
return;
}
};
if params.expected_turn_id.is_empty() {
self.send_invalid_request_error(
request_id,
"expectedTurnId must not be empty".to_string(),
)
.await;
let error = JSONRPCErrorError {
code: INVALID_REQUEST_ERROR_CODE,
message: "expectedTurnId must not be empty".to_string(),
data: None,
};
self.track_error_response(&request_id, &error, /*error_type*/ None);
self.outgoing.send_error(request_id, error).await;
return;
}
self.outgoing
.record_request_turn_id(&request_id, &params.expected_turn_id)
.await;
if let Err(error) = Self::validate_v2_input_limit(&params.input) {
self.track_error_response(
&request_id,
&error,
Some(AnalyticsJsonRpcError::Input(InputError::TooLarge)),
);
self.outgoing.send_error(request_id, error).await;
return;
}
@@ -6914,19 +6949,34 @@ impl CodexMessageProcessor {
{
Ok(turn_id) => {
let response = TurnSteerResponse { turn_id };
if self.config.features.enabled(Feature::GeneralAnalytics) {
self.analytics_events_client.track_response(
request_id.connection_id.0,
ClientResponse::TurnSteer {
request_id: request_id.request_id.clone(),
response: response.clone(),
},
);
}
self.outgoing.send_response(request_id, response).await;
}
Err(err) => {
let (code, message, data) = match err {
let (code, message, data, error_type) = match err {
SteerInputError::NoActiveTurn(_) => (
INVALID_REQUEST_ERROR_CODE,
"no active turn to steer".to_string(),
None,
Some(AnalyticsJsonRpcError::TurnSteer(
TurnSteerRequestError::NoActiveTurn,
)),
),
SteerInputError::ExpectedTurnMismatch { expected, actual } => (
INVALID_REQUEST_ERROR_CODE,
format!("expected active turn id `{expected}` but found `{actual}`"),
None,
Some(AnalyticsJsonRpcError::TurnSteer(
TurnSteerRequestError::ExpectedTurnMismatch,
)),
),
SteerInputError::ActiveTurnNotSteerable { turn_kind } => {
let message = match turn_kind {
@@ -6939,11 +6989,9 @@ impl CodexMessageProcessor {
};
let error = TurnError {
message: message.clone(),
codex_error_info: Some(
AppServerCodexErrorInfo::ActiveTurnNotSteerable {
turn_kind: turn_kind.into(),
},
),
codex_error_info: Some(CodexErrorInfo::ActiveTurnNotSteerable {
turn_kind: turn_kind.into(),
}),
additional_details: None,
};
let data = match serde_json::to_value(error) {
@@ -6956,12 +7004,22 @@ impl CodexMessageProcessor {
None
}
};
(INVALID_REQUEST_ERROR_CODE, message, data)
(
INVALID_REQUEST_ERROR_CODE,
message,
data,
Some(AnalyticsJsonRpcError::TurnSteer(
TurnSteerRequestError::ActiveTurnNotSteerable {
turn_kind: turn_kind.into(),
},
)),
)
}
SteerInputError::EmptyInput => (
INVALID_REQUEST_ERROR_CODE,
"input must not be empty".to_string(),
None,
Some(AnalyticsJsonRpcError::Input(InputError::Empty)),
),
};
let error = JSONRPCErrorError {
@@ -6969,6 +7027,7 @@ impl CodexMessageProcessor {
message,
data,
};
self.track_error_response(&request_id, &error, error_type);
self.outgoing.send_error(request_id, error).await;
}
}

View File

@@ -678,7 +678,8 @@ impl MessageProcessor {
return;
}
if self.config.features.enabled(Feature::GeneralAnalytics)
&& let ClientRequest::TurnStart { request_id, .. } = &codex_request
&& let ClientRequest::TurnStart { request_id, .. }
| ClientRequest::TurnSteer { request_id, .. } = &codex_request
{
self.analytics_events_client.track_request(
connection_id.0,

View File

@@ -6,6 +6,7 @@ use app_test_support::create_mock_responses_server_sequence;
use app_test_support::create_mock_responses_server_sequence_unchecked;
use app_test_support::create_shell_command_sse_response;
use app_test_support::to_response;
use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url;
use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE;
use codex_app_server::INVALID_PARAMS_ERROR_CODE;
use codex_app_server_protocol::JSONRPCError;
@@ -23,6 +24,9 @@ use codex_protocol::user_input::MAX_USER_INPUT_TEXT_CHARS;
use tempfile::TempDir;
use tokio::time::timeout;
use super::analytics::enable_analytics_capture;
use super::analytics::wait_for_analytics_event;
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
#[tokio::test]
@@ -32,7 +36,12 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
std::fs::create_dir(&codex_home)?;
let server = create_mock_responses_server_sequence(vec![]).await;
create_config_toml(&codex_home, &server.uri())?;
write_mock_responses_config_toml_with_chatgpt_base_url(
&codex_home,
&server.uri(),
&server.uri(),
)?;
enable_analytics_capture(&server, &codex_home).await?;
let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
@@ -52,7 +61,7 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
let steer_req = mcp
.send_turn_steer_request(TurnSteerParams {
thread_id: thread.id,
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "steer".to_string(),
text_elements: Vec::new(),
@@ -68,6 +77,21 @@ async fn turn_steer_requires_active_turn() -> Result<()> {
.await??;
assert_eq!(steer_err.error.code, -32600);
let event =
wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?;
assert_eq!(event["event_params"]["thread_id"], thread.id);
assert_eq!(event["event_params"]["result"], "rejected");
assert_eq!(event["event_params"]["num_input_images"], 0);
assert_eq!(
event["event_params"]["expected_turn_id"],
"turn-does-not-exist"
);
assert_eq!(
event["event_params"]["accepted_turn_id"],
serde_json::Value::Null
);
assert_eq!(event["event_params"]["rejection_reason"], "no_active_turn");
Ok(())
}
@@ -96,7 +120,12 @@ async fn turn_steer_rejects_oversized_text_input() -> Result<()> {
"call_sleep",
)?])
.await;
create_config_toml(&codex_home, &server.uri())?;
write_mock_responses_config_toml_with_chatgpt_base_url(
&codex_home,
&server.uri(),
&server.uri(),
)?;
enable_analytics_capture(&server, &codex_home).await?;
let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
@@ -200,7 +229,12 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> {
"call_sleep",
)?])
.await;
create_config_toml(&codex_home, &server.uri())?;
write_mock_responses_config_toml_with_chatgpt_base_url(
&codex_home,
&server.uri(),
&server.uri(),
)?;
enable_analytics_capture(&server, &codex_home).await?;
let mut mcp = McpProcess::new_without_managed_config(&codex_home).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
@@ -261,31 +295,20 @@ async fn turn_steer_returns_active_turn_id() -> Result<()> {
let steer: TurnSteerResponse = to_response::<TurnSteerResponse>(steer_resp)?;
assert_eq!(steer.turn_id, turn.id);
let event =
wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_steer_event").await?;
assert_eq!(event["event_params"]["thread_id"], thread.id);
assert_eq!(event["event_params"]["result"], "accepted");
assert_eq!(event["event_params"]["num_input_images"], 0);
assert_eq!(event["event_params"]["expected_turn_id"], turn.id);
assert_eq!(event["event_params"]["accepted_turn_id"], turn.id);
assert_eq!(
event["event_params"]["rejection_reason"],
serde_json::Value::Null
);
mcp.interrupt_turn_and_wait_for_aborted(thread.id, steer.turn_id, DEFAULT_READ_TIMEOUT)
.await?;
Ok(())
}
fn create_config_toml(codex_home: &std::path::Path, server_uri: &str) -> std::io::Result<()> {
let config_toml = codex_home.join("config.toml");
std::fs::write(
config_toml,
format!(
r#"
model = "mock-model"
approval_policy = "never"
sandbox_mode = "danger-full-access"
model_provider = "mock_provider"
[model_providers.mock_provider]
name = "Mock provider for test"
base_url = "{server_uri}/v1"
wire_api = "responses"
request_max_retries = 0
stream_max_retries = 0
"#
),
)
}