openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-30 19:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Attribute command and plugin analytics to the invoking model (#45445)

Browse Source 
## Why

Command execution and plugin measurement events lack model and reasoning-effort labels. Attribution needs to reflect the step that invoked the command, even when model settings change before a background process finishes.

## What changed

- Add `model_slug` and `reasoning_effort` to command execution and plugin measurement analytics.
- Capture model context from resolved step settings and carry it through execution, approval, Guardian review, and plugin metrics collection.
- Preserve the first command-start model context when subsequent start notifications arrive.
- Keep the carried context out of serialized protocol items and generated schemas.

## Testing

Extend analytics tests to cover model switches before invocation and during background execution, default reasoning effort, Guardian-denied commands, and repeated start notifications retaining the original model context.

GitOrigin-RevId: af90e1c0d39bab625f2e89786085b61a9b96c0ce
This commit is contained in:
jwang-openai
2026-09-14 15:30:22 +00:00
committed by copyberry
parent b6a5d5bb14
commit 4d8eca1ff3
40 changed files with 355 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/analytics/src/analytics_client_tests.rs
+45 -18
View File
@@ -869,6 +869,8 @@ fn plugin_measurements(rows: Vec<PluginMeasurementRow>) -> PluginMeasurementsInp
turn_id: "turn-1".to_string(),
item_id: "item-1".to_string(),
originator: "codex_cli_rs".to_string(),
model_slug: None,
reasoning_effort: None,
plugin_id: "sample@openai-curated".to_string(),
execution_id: "execution-1".to_string(),
operation: "security_scan".to_string(),
@@ -946,6 +948,7 @@ fn sample_command_execution_item_with_id(
duration_ms: Option<i64>,
) -> ThreadItem {
ThreadItem::CommandExecution {
model_context: None,
id: id.to_string(),
plugin_id: None,
script_path: None,
@@ -1790,6 +1793,8 @@ fn command_execution_event_serializes_expected_shape() {
let event = TrackEventRequest::CommandExecution(CodexCommandExecutionEventRequest {
event_type: "codex_command_execution_event",
event_params: CodexCommandExecutionEventParams {
model_slug: None,
reasoning_effort: None,
base: CodexToolItemEventBase {
thread_id: "thread-1".to_string(),
session_id: "session-thread-1".to_string(),
@@ -1848,6 +1853,8 @@ fn command_execution_event_serializes_expected_shape() {
json!({
"event_type": "codex_command_execution_event",
"event_params": {
"model_slug": null,
"reasoning_effort": null,
"thread_id": "thread-1",
"session_id": "session-thread-1",
"turn_id": "turn-1",
@@ -2781,23 +2788,35 @@ async fn item_lifecycle_notifications_publish_command_execution_event() {
&mut events,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
ItemStartedNotification {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
started_at_ms: 1_000,
item: sample_command_execution_item(
CommandExecutionStatus::InProgress,
/*exit_code*/ None,
/*duration_ms*/ None,
),
},
))),
&mut events,
)
.await;
for model in ["invoking-model", "later-model"] {
reducer
.ingest(
AnalyticsFact::Notification(Box::new(ServerNotification::ItemStarted(
ItemStartedNotification {
thread_id: "thread-1".to_string(),
turn_id: "turn-1".to_string(),
started_at_ms: 1_000,
item: {
let mut item = sample_command_execution_item(
CommandExecutionStatus::InProgress,
/*exit_code*/ None,
/*duration_ms*/ None,
);
if let ThreadItem::CommandExecution { model_context, .. } = &mut item {
*model_context =
Some(codex_protocol::items::ModelInvocationContext {
model_slug: model.to_string(),
reasoning_effort: Some("max".to_string()),
});
}
item
},
},
))),
&mut events,
)
.await;
}
assert!(
events.is_empty(),
"tool item event should emit on completion"
@@ -2844,6 +2863,8 @@ async fn item_lifecycle_notifications_publish_command_execution_event() {
let payload = serde_json::to_value(&events).expect("serialize events");
assert_eq!(payload.as_array().expect("events array").len(), 1);
assert_eq!(payload[0]["event_params"]["model_slug"], "invoking-model");
assert_eq!(payload[0]["event_params"]["reasoning_effort"], "max");
assert_eq!(payload[0]["event_type"], "codex_command_execution_event");
assert_eq!(payload[0]["event_params"]["thread_id"], "thread-1");
assert_eq!(payload[0]["event_params"]["session_id"], "session-thread-1");
@@ -2899,7 +2920,7 @@ async fn plugin_measurement_batch_emits_directly_and_filters_invalid_rows() {
for index in 0..9 {
too_many_dimensions.insert(format!("dimension_{index}"), "allowed".to_string());
}
let measurements = plugin_measurements(vec![
let mut measurements = plugin_measurements(vec![
PluginMeasurementRow {
measurement_name: "finding_count".to_string(),
number_value: 3.0,
@@ -2921,6 +2942,8 @@ async fn plugin_measurement_batch_emits_directly_and_filters_invalid_rows() {
dimensions: BTreeMap::new(),
},
]);
measurements.model_slug = Some("invoking-model".to_string());
measurements.reasoning_effort = Some("max".to_string());
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::PluginMeasurements(measurements)),
@@ -2942,6 +2965,8 @@ async fn plugin_measurement_batch_emits_directly_and_filters_invalid_rows() {
"operation": "security_scan",
"measurement_name": "finding_count",
"originator": "codex_cli_rs",
"model_slug": "invoking-model",
"reasoning_effort": "max",
"number_value": 3.0,
"dimensions": {"severity": "high"},
},
@@ -2957,6 +2982,8 @@ async fn plugin_measurement_batch_emits_directly_and_filters_invalid_rows() {
"operation": "security_scan",
"measurement_name": "files_scanned",
"originator": "codex_cli_rs",
"model_slug": "invoking-model",
"reasoning_effort": "max",
"number_value": 17.0,
"dimensions": null,
},
codex-rs/analytics/src/client_tests.rs
+4
View File
@@ -420,6 +420,8 @@ async fn api_key_auth_sends_only_plugin_events_to_codex_backend() {
TrackEventRequest::PluginMeasurement(CodexPluginMeasurementEventRequest {
event_type: "codex_plugin_measurement_event",
event_params: CodexPluginMeasurementEventParams {
model_slug: None,
reasoning_effort: None,
thread_id: thread_id.to_string(),
turn_id: "turn-1".to_string(),
item_id: "item-1".to_string(),
@@ -712,6 +714,8 @@ fn track_plugin_measurements_rejects_unbounded_inputs_before_queueing() {
turn_id: "turn-1".to_string(),
item_id: "item-1".to_string(),
originator: "codex_cli_rs".to_string(),
model_slug: None,
reasoning_effort: None,
plugin_id: "sample@openai-curated".to_string(),
execution_id: "execution-1".to_string(),
operation: "security_scan".to_string(),
codex-rs/analytics/src/events.rs
+4
View File
@@ -778,6 +778,8 @@ pub(crate) enum WebSearchActionKind {
#[derive(Serialize)]
pub(crate) struct CodexCommandExecutionEventParams {
pub(crate) model_slug: Option<String>,
pub(crate) reasoning_effort: Option<String>,
#[serde(flatten)]
pub(crate) base: CodexToolItemEventBase,
pub(crate) plugin_id: Option<String>,
@@ -803,6 +805,8 @@ pub(crate) struct CodexPluginMeasurementEventParams {
pub(crate) turn_id: String,
pub(crate) item_id: String,
pub(crate) originator: String,
pub(crate) model_slug: Option<String>,
pub(crate) reasoning_effort: Option<String>,
pub(crate) plugin_id: String,
pub(crate) execution_id: String,
pub(crate) operation: String,
codex-rs/analytics/src/facts.rs
+2
View File
@@ -604,6 +604,8 @@ pub struct PluginMeasurementsInput {
pub turn_id: String,
pub item_id: String,
pub originator: String,
pub model_slug: Option<String>,
pub reasoning_effort: Option<String>,
pub plugin_id: String,
pub execution_id: String,
pub operation: String,
codex-rs/analytics/src/reducer.rs
+24 -7
View File
@@ -159,6 +159,7 @@ use codex_login::default_client::originator;
use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::items::ModelInvocationContext;
use codex_protocol::items::is_safe_plugin_relative_path;
use codex_protocol::models::PermissionProfile;
use codex_protocol::protocol::SessionSource;
@@ -203,7 +204,7 @@ pub(crate) struct AnalyticsReducer {
turns: HashMap<String, TurnState>,
connections: HashMap<u64, ConnectionState>,
threads: HashMap<String, ThreadAnalyticsState>,
tool_items_started_at_ms: HashMap<ToolItemKey, u64>,
tool_items_started_at_ms: HashMap<ToolItemKey, (u64, Option<ModelInvocationContext>)>,
tool_response_states: HashMap<(String, String), ToolResponseState>,
code_mode_cells: HashMap<String, HashMap<String, CodeModeCellState>>,
pending_reviews: HashMap<RequestId, PendingReviewState>,
@@ -1891,14 +1892,18 @@ impl AnalyticsReducer {
else {
return;
};
self.tool_items_started_at_ms.insert(
ToolItemKey {
let model_context = match &notification.item {
ThreadItem::CommandExecution { model_context, .. } => model_context.clone(),
_ => None,
};
self.tool_items_started_at_ms
.entry(ToolItemKey {
thread_id: notification.thread_id,
turn_id: notification.turn_id,
item_id: item_id.to_string(),
},
started_at_ms,
);
})
.and_modify(|(timestamp, _)| *timestamp = started_at_ms)
.or_insert((started_at_ms, model_context));
}
ServerNotification::ItemCompleted(notification) => {
if matches!(notification.item, ThreadItem::SubAgentActivity { .. }) {
@@ -1931,7 +1936,9 @@ impl AnalyticsReducer {
turn_id: notification.turn_id.clone(),
item_id: item_id.to_string(),
};
let Some(started_at_ms) = self.tool_items_started_at_ms.remove(&key) else {
let Some((started_at_ms, model_context)) =
self.tool_items_started_at_ms.remove(&key)
else {
tracing::warn!(
thread_id = %notification.thread_id,
turn_id = %notification.turn_id,
@@ -1953,6 +1960,7 @@ impl AnalyticsReducer {
thread_id: &notification.thread_id,
turn_id: &notification.turn_id,
item: &notification.item,
model_context: model_context.as_ref(),
started_at_ms,
completed_at_ms,
connection_state,
@@ -2063,6 +2071,8 @@ impl AnalyticsReducer {
turn_id,
item_id,
originator,
model_slug,
reasoning_effort,
plugin_id,
execution_id,
operation,
@@ -2079,6 +2089,8 @@ impl AnalyticsReducer {
turn_id: turn_id.clone(),
item_id: item_id.clone(),
originator: originator.clone(),
model_slug: model_slug.clone(),
reasoning_effort: reasoning_effort.clone(),
plugin_id: plugin_id.clone(),
execution_id: execution_id.clone(),
operation: operation.clone(),
@@ -2598,6 +2610,7 @@ struct ToolItemEventInput<'a> {
thread_id: &'a str,
turn_id: &'a str,
item: &'a ThreadItem,
model_context: Option<&'a ModelInvocationContext>,
started_at_ms: u64,
completed_at_ms: u64,
connection_state: &'a ConnectionState,
@@ -2611,6 +2624,7 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option<TrackEventRequest> {
thread_id,
turn_id,
item,
model_context,
started_at_ms,
completed_at_ms,
connection_state,
@@ -2655,6 +2669,9 @@ fn tool_item_event(input: ToolItemEventInput<'_>) -> Option<TrackEventRequest> {
CodexCommandExecutionEventRequest {
event_type: "codex_command_execution_event",
event_params: CodexCommandExecutionEventParams {
model_slug: model_context.map(|context| context.model_slug.clone()),
reasoning_effort: model_context
.and_then(|context| context.reasoning_effort.clone()),
base,
plugin_id: plugin_id.clone(),
script_path: safe_plugin_relative_script_path(