Attribute command and plugin analytics to the invoking model (#45445)

## Why

Command execution and plugin measurement events lack model and reasoning-effort labels. Attribution needs to reflect the step that invoked the command, even when model settings change before a background process finishes.

## What changed

- Add `model_slug` and `reasoning_effort` to command execution and plugin measurement analytics.
- Capture model context from resolved step settings and carry it through execution, approval, Guardian review, and plugin metrics collection.
- Preserve the first command-start model context when subsequent start notifications arrive.
- Keep the carried context out of serialized protocol items and generated schemas.

## Testing

Extend analytics tests to cover model switches before invocation and during background execution, default reasoning effort, Guardian-denied commands, and repeated start notifications retaining the original model context.

GitOrigin-RevId: af90e1c0d39bab625f2e89786085b61a9b96c0ce
This commit is contained in:
jwang-openai
2026-09-14 15:30:22 +00:00
committed by copyberry
parent b6a5d5bb14
commit 4d8eca1ff3
40 changed files with 355 additions and 43 deletions
@@ -92,6 +92,7 @@ use codex_core::ThreadManager;
use codex_features::Feature;
use codex_protocol::ThreadId;
use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool;
use codex_protocol::items::ModelInvocationContext;
use codex_protocol::items::TurnItem as CoreTurnItem;
use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile;
use codex_protocol::plan_tool::UpdatePlanArgs;
@@ -129,6 +130,7 @@ enum CommandExecutionApprovalPresentation {
#[derive(Debug, PartialEq)]
struct CommandExecutionCompletionItem {
model_context: Option<ModelInvocationContext>,
plugin_id: Option<String>,
script_path: Option<String>,
command: String,
@@ -294,6 +296,7 @@ pub(crate) async fn apply_bespoke_event_handling(
) {
Some(ThreadItem::CommandExecution {
id,
model_context,
plugin_id,
script_path,
command,
@@ -303,6 +306,7 @@ pub(crate) async fn apply_bespoke_event_handling(
}) => Some((
id,
CommandExecutionCompletionItem {
model_context,
plugin_id,
script_path,
command,
@@ -324,6 +328,7 @@ pub(crate) async fn apply_bespoke_event_handling(
&conversation_id,
assessment_turn_id.clone(),
target_item_id.clone(),
completion_item.model_context.clone(),
completion_item.plugin_id.clone(),
completion_item.script_path.clone(),
completion_item.command.clone(),
@@ -669,6 +674,7 @@ pub(crate) async fn apply_bespoke_event_handling(
.map(CommandExecutionApprovalDecision::from)
.collect::<Vec<_>>();
let ExecApprovalRequestEvent {
model_context,
kind,
call_id,
plugin_id,
@@ -723,6 +729,7 @@ pub(crate) async fn apply_bespoke_event_handling(
CommandExecutionApprovalPresentation::Network(network_approval_context)
} else {
let completion_item = CommandExecutionCompletionItem {
model_context,
plugin_id,
script_path,
command: command_presentation.command,
@@ -751,6 +758,7 @@ pub(crate) async fn apply_bespoke_event_handling(
&conversation_id,
event_turn_id.clone(),
call_id.clone(),
completion_item.model_context.clone(),
completion_item.plugin_id.clone(),
completion_item.script_path.clone(),
completion_item.command.clone(),
@@ -1389,6 +1397,7 @@ async fn start_command_execution_item(
conversation_id: &ThreadId,
turn_id: String,
item_id: String,
model_context: Option<ModelInvocationContext>,
plugin_id: Option<String>,
script_path: Option<String>,
command: String,
@@ -1412,6 +1421,7 @@ async fn start_command_execution_item(
started_at_ms: now_unix_timestamp_ms(),
item: ThreadItem::CommandExecution {
id: item_id,
model_context,
plugin_id,
script_path,
command,
@@ -1456,6 +1466,7 @@ async fn complete_command_execution_item(
let item = ThreadItem::CommandExecution {
id: item_id,
model_context: completion_item.model_context,
plugin_id: completion_item.plugin_id,
script_path: completion_item.script_path,
command: completion_item.command,
@@ -2183,6 +2194,7 @@ mod tests {
fn command_execution_completion_item(command: &str) -> CommandExecutionCompletionItem {
CommandExecutionCompletionItem {
model_context: None,
plugin_id: Some("sample@openai-curated".to_string()),
script_path: Some("scripts/run.py".to_string()),
command: command.to_string(),
@@ -2217,6 +2229,7 @@ mod tests {
};
GuardianAssessmentEvent {
review_reason: None,
model_context: None,
id: format!("review-{id}"),
target_item_id: Some(id.to_string()),
plugin_id: Some("sample@openai-curated".to_string()),
@@ -2285,6 +2298,7 @@ mod tests {
"turn-from-event",
&GuardianAssessmentEvent {
review_reason: None,
model_context: None,
id: "review-1".to_string(),
target_item_id: Some("item-1".to_string()),
plugin_id: None,
@@ -2334,6 +2348,7 @@ mod tests {
"turn-from-event",
&GuardianAssessmentEvent {
review_reason: None,
model_context: None,
id: "review-2".to_string(),
target_item_id: Some("item-2".to_string()),
plugin_id: None,
@@ -2391,6 +2406,7 @@ mod tests {
"turn-from-event",
&GuardianAssessmentEvent {
review_reason: None,
model_context: None,
id: "review-3".to_string(),
target_item_id: None,
plugin_id: None,
@@ -2446,6 +2462,7 @@ mod tests {
&conversation_id,
"turn-1".to_string(),
"cmd-1".to_string(),
completion_item.model_context.clone(),
completion_item.plugin_id.clone(),
completion_item.script_path.clone(),
completion_item.command.clone(),
@@ -2466,6 +2483,7 @@ mod tests {
assert_eq!(
payload.item,
ThreadItem::CommandExecution {
model_context: None,
id: "cmd-1".to_string(),
plugin_id: completion_item.plugin_id.clone(),
script_path: completion_item.script_path.clone(),
@@ -2488,6 +2506,7 @@ mod tests {
&conversation_id,
"turn-1".to_string(),
"cmd-1".to_string(),
completion_item.model_context.clone(),
completion_item.plugin_id.clone(),
completion_item.script_path.clone(),
completion_item.command.clone(),
@@ -2524,6 +2543,7 @@ mod tests {
&conversation_id,
"turn-1".to_string(),
"cmd-1".to_string(),
completion_item.model_context.clone(),
completion_item.plugin_id.clone(),
completion_item.script_path.clone(),
completion_item.command.clone(),
+123 -16
View File
@@ -14,11 +14,15 @@ use codex_app_server_protocol::ClientRequest;
use codex_app_server_protocol::JSONRPCResponse;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::SandboxPolicy;
use codex_app_server_protocol::ServerRequest;
use codex_app_server_protocol::ThreadItem;
use codex_app_server_protocol::ThreadReadParams;
use codex_app_server_protocol::ThreadReadResponse;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::TurnSettingsUpdateParams;
use codex_app_server_protocol::TurnSettingsUpdateResponse;
use codex_app_server_protocol::TurnSettingsUpdateStatus;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::TurnStartResponse;
use codex_app_server_protocol::UserInput;
@@ -29,7 +33,10 @@ use codex_core::config::ConfigBuilder;
use codex_core_plugins::loader::curated_plugin_cache_version;
use codex_core_plugins::store::PluginStore;
use codex_features::Feature;
use codex_models_manager::bundled_models_response;
use codex_plugin::PluginId;
use codex_protocol::openai_models::ModelsResponse;
use codex_protocol::openai_models::ReasoningEffort;
use core_test_support::responses;
use core_test_support::skip_if_no_network;
use core_test_support::skip_if_remote;
@@ -135,6 +142,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
text: "Review three commands".to_string(),
text_elements: Vec::new(),
}],
effort: Some(ReasoningEffort::High),
..Default::default()
})
.await?;
@@ -166,6 +174,19 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
timeout(READ_TIMEOUT, app_server.shutdown_gracefully()).await??;
let events = captured_analytics_events(&server).await;
assert!(!serde_json::to_string(&events)?.contains(PRIVATE));
let parent_command = events
.iter()
.find(|event| {
event["event_type"] == "codex_command_execution_event"
&& event["event_params"]["thread_id"] == thread.id
&& event["event_params"]["item_id"] == "parent-first"
})
.expect("denied parent command analytics");
let params = &parent_command["event_params"];
assert_eq!(
json!([params["model_slug"], params["reasoning_effort"]]),
json!(["mock-model", "high"])
);
let children = events
.iter()
.filter(|event| event["event_params"]["subagent_source"] == "guardian")
@@ -692,6 +713,7 @@ operations:
&script_path,
r#"test -n "$CODEX_PLUGIN_METRICS_OUTPUT"
sleep "${1:-0.3}"
while [ -n "${2:-}" ] && [ ! -f "$2" ]; do sleep 0.01; done
printf '%s' '{"version":1,"measurements":[{"name":"findings","value":3,"dimensions":{"severity":"high"}},{"name":"files_scanned","value":17}]}' > "$CODEX_PLUGIN_METRICS_OUTPUT"
"#,
)?;
@@ -724,12 +746,14 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
let codex_home = TempDir::new()?;
let script_path = write_curated_metrics_plugin(codex_home.path())?.canonicalize()?;
let release_path = codex_home.path().join("release-command");
let mut command = vec![
"/bin/sh".to_string(),
script_path.to_string_lossy().into_owned(),
];
if background {
command.push("1.0".to_string());
command.push(release_path.to_string_lossy().into_owned());
}
let call_id = "curated-plugin-metrics";
let arguments = serde_json::to_string(&json!({
@@ -742,8 +766,32 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
responses::ev_completed("resp-1"),
]);
let final_response = create_final_assistant_message_sse_response("done")?;
let server =
create_mock_responses_server_sequence(vec![command_response, final_response]).await;
let pause_response = |id: &str| {
responses::sse(vec![
responses::ev_response_created(id),
responses::ev_function_call(
id,
"request_user_input",
&json!({
"questions": [{
"id": "continue", "header": "Continue", "question": "Continue?",
"options": [
{"label": "Yes", "description": "Continue the turn."},
{"label": "No", "description": "Stop the turn."}
]
}]
})
.to_string(),
),
responses::ev_completed(id),
])
};
let mut response_sequence = vec![pause_response("before-command"), command_response];
if background {
response_sequence.push(pause_response("after-launch"));
}
response_sequence.push(final_response);
let server = create_mock_responses_server_sequence(response_sequence).await;
let analytics_server = responses::start_mock_server().await;
write_mock_responses_config_toml_with_chatgpt_base_url(
@@ -753,11 +801,40 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
)?;
let config_path = codex_home.path().join("config.toml");
let config = std::fs::read_to_string(&config_path)?;
let model = bundled_models_response()?
.models
.into_iter()
.find(|model| model.slug == "gpt-5.4")
.expect("bundled gpt-5.4 model");
let models = [
("initial-model", ReasoningEffort::Low),
("invoking-model", ReasoningEffort::High),
]
.into_iter()
.map(|(slug, effort)| {
let mut model = model.clone();
model.slug = slug.to_string();
model.default_reasoning_level = Some(effort);
model
})
.collect();
let catalog_path = codex_home.path().join("measurement-models.json");
std::fs::write(
&catalog_path,
serde_json::to_vec(&ModelsResponse { models })?,
)?;
let catalog_config = format!(
"model_catalog_json = {}",
serde_json::to_string(&catalog_path)?
);
std::fs::write(
config_path,
format!(
r#"{config}
r#"{catalog_config}
{config}
[features]
step_model_switching = true
default_mode_request_user_input = true
plugins = true
remote_plugin = false
unified_exec = true
@@ -787,7 +864,7 @@ enabled = true
timeout(Duration::from_secs(10), mcp.initialize()).await??;
let thread_request = mcp
.send_thread_start_request_with_auto_env(ThreadStartParams {
model: Some("mock-model".to_string()),
model: Some("initial-model".to_string()),
service_name: Some("codex_work_desktop".to_string()),
..Default::default()
})
@@ -813,11 +890,43 @@ enabled = true
..Default::default()
})
.await?;
timeout(
let turn_response = timeout(
Duration::from_secs(10),
mcp.read_stream_until_response_message(RequestId::Integer(turn_request)),
)
.await??;
let turn_response: TurnStartResponse = to_response(turn_response)?;
for model in if background {
vec!["invoking-model", "initial-model"]
} else {
vec!["invoking-model"]
} {
let request = timeout(
Duration::from_secs(10),
mcp.read_stream_until_request_message(),
)
.await??;
let ServerRequest::ToolRequestUserInput { request_id, .. } = request else {
anyhow::bail!("expected request_user_input, received {request:?}");
};
let response: TurnSettingsUpdateResponse = mcp
.request(|request_id| ClientRequest::TurnSettingsUpdate {
request_id,
params: TurnSettingsUpdateParams {
thread_id: thread_id.clone(),
turn_id: turn_response.turn.id.clone(),
model: Some(model.to_string()),
..Default::default()
},
})
.await?;
assert_eq!(response.status, TurnSettingsUpdateStatus::Applied);
mcp.send_response(
request_id,
json!({"answers": {"continue": {"answers": ["Yes"]}}}),
)
.await?;
}
let completed_turn = timeout(
Duration::from_secs(10),
mcp.read_stream_until_notification_message("turn/completed"),
@@ -830,17 +939,7 @@ enabled = true
.expect("completed turn id");
if background {
let model_request_bodies = server
.received_requests()
.await
.unwrap_or_default()
.into_iter()
.filter(|request| request.url.path().ends_with("/responses"))
.map(|request| serde_json::from_slice::<Value>(&request.body))
.collect::<Result<Vec<_>, _>>()?;
let request_text = serde_json::to_string(&model_request_bodies)?;
assert!(request_text.contains("Process running with session ID "));
assert!(!request_text.contains("Process exited with code 0"));
std::fs::write(release_path, "release")?;
}
for measurement_name in ["findings", "files_scanned"] {
@@ -859,12 +958,16 @@ enabled = true
.await?;
assert_eq!(
json!({
"model_slug": command_event["event_params"]["model_slug"],
"reasoning_effort": command_event["event_params"]["reasoning_effort"],
"plugin_id": command_event["event_params"]["plugin_id"],
"script_path": command_event["event_params"]["script_path"],
"item_id": command_event["event_params"]["item_id"],
"exit_code": command_event["event_params"]["exit_code"],
}),
json!({
"model_slug": "invoking-model",
"reasoning_effort": "high",
"plugin_id": METRICS_PLUGIN_ID,
"script_path": "scripts/run.sh",
"item_id": call_id,
@@ -952,10 +1055,12 @@ enabled = true
"execution_id",
"item_id",
"measurement_name",
"model_slug",
"number_value",
"operation",
"originator",
"plugin_id",
"reasoning_effort",
"thread_id",
"turn_id",
]
@@ -963,6 +1068,8 @@ enabled = true
assert_eq!(event_params["thread_id"], thread_id);
assert_eq!(event_params["turn_id"], turn_id);
assert_eq!(event_params["originator"], "codex_work_desktop");
assert_eq!(event_params["model_slug"], "invoking-model");
assert_eq!(event_params["reasoning_effort"], "high");
}
Ok(())