mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Attribute command and plugin analytics to the invoking model (#45445)
## Why Command execution and plugin measurement events lack model and reasoning-effort labels. Attribution needs to reflect the step that invoked the command, even when model settings change before a background process finishes. ## What changed - Add `model_slug` and `reasoning_effort` to command execution and plugin measurement analytics. - Capture model context from resolved step settings and carry it through execution, approval, Guardian review, and plugin metrics collection. - Preserve the first command-start model context when subsequent start notifications arrive. - Keep the carried context out of serialized protocol items and generated schemas. ## Testing Extend analytics tests to cover model switches before invocation and during background execution, default reasoning effort, Guardian-denied commands, and repeated start notifications retaining the original model context. GitOrigin-RevId: af90e1c0d39bab625f2e89786085b61a9b96c0ce
This commit is contained in:
@@ -92,6 +92,7 @@ use codex_core::ThreadManager;
|
||||
use codex_features::Feature;
|
||||
use codex_protocol::ThreadId;
|
||||
use codex_protocol::items::CollabAgentTool as CoreCollabAgentTool;
|
||||
use codex_protocol::items::ModelInvocationContext;
|
||||
use codex_protocol::items::TurnItem as CoreTurnItem;
|
||||
use codex_protocol::models::AdditionalPermissionProfile as CoreAdditionalPermissionProfile;
|
||||
use codex_protocol::plan_tool::UpdatePlanArgs;
|
||||
@@ -129,6 +130,7 @@ enum CommandExecutionApprovalPresentation {
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct CommandExecutionCompletionItem {
|
||||
model_context: Option<ModelInvocationContext>,
|
||||
plugin_id: Option<String>,
|
||||
script_path: Option<String>,
|
||||
command: String,
|
||||
@@ -294,6 +296,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
) {
|
||||
Some(ThreadItem::CommandExecution {
|
||||
id,
|
||||
model_context,
|
||||
plugin_id,
|
||||
script_path,
|
||||
command,
|
||||
@@ -303,6 +306,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
}) => Some((
|
||||
id,
|
||||
CommandExecutionCompletionItem {
|
||||
model_context,
|
||||
plugin_id,
|
||||
script_path,
|
||||
command,
|
||||
@@ -324,6 +328,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
&conversation_id,
|
||||
assessment_turn_id.clone(),
|
||||
target_item_id.clone(),
|
||||
completion_item.model_context.clone(),
|
||||
completion_item.plugin_id.clone(),
|
||||
completion_item.script_path.clone(),
|
||||
completion_item.command.clone(),
|
||||
@@ -669,6 +674,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
.map(CommandExecutionApprovalDecision::from)
|
||||
.collect::<Vec<_>>();
|
||||
let ExecApprovalRequestEvent {
|
||||
model_context,
|
||||
kind,
|
||||
call_id,
|
||||
plugin_id,
|
||||
@@ -723,6 +729,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
CommandExecutionApprovalPresentation::Network(network_approval_context)
|
||||
} else {
|
||||
let completion_item = CommandExecutionCompletionItem {
|
||||
model_context,
|
||||
plugin_id,
|
||||
script_path,
|
||||
command: command_presentation.command,
|
||||
@@ -751,6 +758,7 @@ pub(crate) async fn apply_bespoke_event_handling(
|
||||
&conversation_id,
|
||||
event_turn_id.clone(),
|
||||
call_id.clone(),
|
||||
completion_item.model_context.clone(),
|
||||
completion_item.plugin_id.clone(),
|
||||
completion_item.script_path.clone(),
|
||||
completion_item.command.clone(),
|
||||
@@ -1389,6 +1397,7 @@ async fn start_command_execution_item(
|
||||
conversation_id: &ThreadId,
|
||||
turn_id: String,
|
||||
item_id: String,
|
||||
model_context: Option<ModelInvocationContext>,
|
||||
plugin_id: Option<String>,
|
||||
script_path: Option<String>,
|
||||
command: String,
|
||||
@@ -1412,6 +1421,7 @@ async fn start_command_execution_item(
|
||||
started_at_ms: now_unix_timestamp_ms(),
|
||||
item: ThreadItem::CommandExecution {
|
||||
id: item_id,
|
||||
model_context,
|
||||
plugin_id,
|
||||
script_path,
|
||||
command,
|
||||
@@ -1456,6 +1466,7 @@ async fn complete_command_execution_item(
|
||||
|
||||
let item = ThreadItem::CommandExecution {
|
||||
id: item_id,
|
||||
model_context: completion_item.model_context,
|
||||
plugin_id: completion_item.plugin_id,
|
||||
script_path: completion_item.script_path,
|
||||
command: completion_item.command,
|
||||
@@ -2183,6 +2194,7 @@ mod tests {
|
||||
|
||||
fn command_execution_completion_item(command: &str) -> CommandExecutionCompletionItem {
|
||||
CommandExecutionCompletionItem {
|
||||
model_context: None,
|
||||
plugin_id: Some("sample@openai-curated".to_string()),
|
||||
script_path: Some("scripts/run.py".to_string()),
|
||||
command: command.to_string(),
|
||||
@@ -2217,6 +2229,7 @@ mod tests {
|
||||
};
|
||||
GuardianAssessmentEvent {
|
||||
review_reason: None,
|
||||
model_context: None,
|
||||
id: format!("review-{id}"),
|
||||
target_item_id: Some(id.to_string()),
|
||||
plugin_id: Some("sample@openai-curated".to_string()),
|
||||
@@ -2285,6 +2298,7 @@ mod tests {
|
||||
"turn-from-event",
|
||||
&GuardianAssessmentEvent {
|
||||
review_reason: None,
|
||||
model_context: None,
|
||||
id: "review-1".to_string(),
|
||||
target_item_id: Some("item-1".to_string()),
|
||||
plugin_id: None,
|
||||
@@ -2334,6 +2348,7 @@ mod tests {
|
||||
"turn-from-event",
|
||||
&GuardianAssessmentEvent {
|
||||
review_reason: None,
|
||||
model_context: None,
|
||||
id: "review-2".to_string(),
|
||||
target_item_id: Some("item-2".to_string()),
|
||||
plugin_id: None,
|
||||
@@ -2391,6 +2406,7 @@ mod tests {
|
||||
"turn-from-event",
|
||||
&GuardianAssessmentEvent {
|
||||
review_reason: None,
|
||||
model_context: None,
|
||||
id: "review-3".to_string(),
|
||||
target_item_id: None,
|
||||
plugin_id: None,
|
||||
@@ -2446,6 +2462,7 @@ mod tests {
|
||||
&conversation_id,
|
||||
"turn-1".to_string(),
|
||||
"cmd-1".to_string(),
|
||||
completion_item.model_context.clone(),
|
||||
completion_item.plugin_id.clone(),
|
||||
completion_item.script_path.clone(),
|
||||
completion_item.command.clone(),
|
||||
@@ -2466,6 +2483,7 @@ mod tests {
|
||||
assert_eq!(
|
||||
payload.item,
|
||||
ThreadItem::CommandExecution {
|
||||
model_context: None,
|
||||
id: "cmd-1".to_string(),
|
||||
plugin_id: completion_item.plugin_id.clone(),
|
||||
script_path: completion_item.script_path.clone(),
|
||||
@@ -2488,6 +2506,7 @@ mod tests {
|
||||
&conversation_id,
|
||||
"turn-1".to_string(),
|
||||
"cmd-1".to_string(),
|
||||
completion_item.model_context.clone(),
|
||||
completion_item.plugin_id.clone(),
|
||||
completion_item.script_path.clone(),
|
||||
completion_item.command.clone(),
|
||||
@@ -2524,6 +2543,7 @@ mod tests {
|
||||
&conversation_id,
|
||||
"turn-1".to_string(),
|
||||
"cmd-1".to_string(),
|
||||
completion_item.model_context.clone(),
|
||||
completion_item.plugin_id.clone(),
|
||||
completion_item.script_path.clone(),
|
||||
completion_item.command.clone(),
|
||||
|
||||
@@ -14,11 +14,15 @@ use codex_app_server_protocol::ClientRequest;
|
||||
use codex_app_server_protocol::JSONRPCResponse;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::SandboxPolicy;
|
||||
use codex_app_server_protocol::ServerRequest;
|
||||
use codex_app_server_protocol::ThreadItem;
|
||||
use codex_app_server_protocol::ThreadReadParams;
|
||||
use codex_app_server_protocol::ThreadReadResponse;
|
||||
use codex_app_server_protocol::ThreadStartParams;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
use codex_app_server_protocol::TurnSettingsUpdateParams;
|
||||
use codex_app_server_protocol::TurnSettingsUpdateResponse;
|
||||
use codex_app_server_protocol::TurnSettingsUpdateStatus;
|
||||
use codex_app_server_protocol::TurnStartParams;
|
||||
use codex_app_server_protocol::TurnStartResponse;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
@@ -29,7 +33,10 @@ use codex_core::config::ConfigBuilder;
|
||||
use codex_core_plugins::loader::curated_plugin_cache_version;
|
||||
use codex_core_plugins::store::PluginStore;
|
||||
use codex_features::Feature;
|
||||
use codex_models_manager::bundled_models_response;
|
||||
use codex_plugin::PluginId;
|
||||
use codex_protocol::openai_models::ModelsResponse;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::skip_if_remote;
|
||||
@@ -135,6 +142,7 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
|
||||
text: "Review three commands".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
effort: Some(ReasoningEffort::High),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
@@ -166,6 +174,19 @@ async fn guardian_review_turns_and_tools_reach_analytics() -> Result<()> {
|
||||
timeout(READ_TIMEOUT, app_server.shutdown_gracefully()).await??;
|
||||
let events = captured_analytics_events(&server).await;
|
||||
assert!(!serde_json::to_string(&events)?.contains(PRIVATE));
|
||||
let parent_command = events
|
||||
.iter()
|
||||
.find(|event| {
|
||||
event["event_type"] == "codex_command_execution_event"
|
||||
&& event["event_params"]["thread_id"] == thread.id
|
||||
&& event["event_params"]["item_id"] == "parent-first"
|
||||
})
|
||||
.expect("denied parent command analytics");
|
||||
let params = &parent_command["event_params"];
|
||||
assert_eq!(
|
||||
json!([params["model_slug"], params["reasoning_effort"]]),
|
||||
json!(["mock-model", "high"])
|
||||
);
|
||||
let children = events
|
||||
.iter()
|
||||
.filter(|event| event["event_params"]["subagent_source"] == "guardian")
|
||||
@@ -692,6 +713,7 @@ operations:
|
||||
&script_path,
|
||||
r#"test -n "$CODEX_PLUGIN_METRICS_OUTPUT"
|
||||
sleep "${1:-0.3}"
|
||||
while [ -n "${2:-}" ] && [ ! -f "$2" ]; do sleep 0.01; done
|
||||
printf '%s' '{"version":1,"measurements":[{"name":"findings","value":3,"dimensions":{"severity":"high"}},{"name":"files_scanned","value":17}]}' > "$CODEX_PLUGIN_METRICS_OUTPUT"
|
||||
"#,
|
||||
)?;
|
||||
@@ -724,12 +746,14 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
let script_path = write_curated_metrics_plugin(codex_home.path())?.canonicalize()?;
|
||||
let release_path = codex_home.path().join("release-command");
|
||||
let mut command = vec![
|
||||
"/bin/sh".to_string(),
|
||||
script_path.to_string_lossy().into_owned(),
|
||||
];
|
||||
if background {
|
||||
command.push("1.0".to_string());
|
||||
command.push(release_path.to_string_lossy().into_owned());
|
||||
}
|
||||
let call_id = "curated-plugin-metrics";
|
||||
let arguments = serde_json::to_string(&json!({
|
||||
@@ -742,8 +766,32 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
|
||||
responses::ev_completed("resp-1"),
|
||||
]);
|
||||
let final_response = create_final_assistant_message_sse_response("done")?;
|
||||
let server =
|
||||
create_mock_responses_server_sequence(vec![command_response, final_response]).await;
|
||||
let pause_response = |id: &str| {
|
||||
responses::sse(vec![
|
||||
responses::ev_response_created(id),
|
||||
responses::ev_function_call(
|
||||
id,
|
||||
"request_user_input",
|
||||
&json!({
|
||||
"questions": [{
|
||||
"id": "continue", "header": "Continue", "question": "Continue?",
|
||||
"options": [
|
||||
{"label": "Yes", "description": "Continue the turn."},
|
||||
{"label": "No", "description": "Stop the turn."}
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
responses::ev_completed(id),
|
||||
])
|
||||
};
|
||||
let mut response_sequence = vec![pause_response("before-command"), command_response];
|
||||
if background {
|
||||
response_sequence.push(pause_response("after-launch"));
|
||||
}
|
||||
response_sequence.push(final_response);
|
||||
let server = create_mock_responses_server_sequence(response_sequence).await;
|
||||
|
||||
let analytics_server = responses::start_mock_server().await;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
@@ -753,11 +801,40 @@ async fn assert_plugin_measurement_analytics(remote: bool, background: bool) ->
|
||||
)?;
|
||||
let config_path = codex_home.path().join("config.toml");
|
||||
let config = std::fs::read_to_string(&config_path)?;
|
||||
let model = bundled_models_response()?
|
||||
.models
|
||||
.into_iter()
|
||||
.find(|model| model.slug == "gpt-5.4")
|
||||
.expect("bundled gpt-5.4 model");
|
||||
let models = [
|
||||
("initial-model", ReasoningEffort::Low),
|
||||
("invoking-model", ReasoningEffort::High),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(slug, effort)| {
|
||||
let mut model = model.clone();
|
||||
model.slug = slug.to_string();
|
||||
model.default_reasoning_level = Some(effort);
|
||||
model
|
||||
})
|
||||
.collect();
|
||||
let catalog_path = codex_home.path().join("measurement-models.json");
|
||||
std::fs::write(
|
||||
&catalog_path,
|
||||
serde_json::to_vec(&ModelsResponse { models })?,
|
||||
)?;
|
||||
let catalog_config = format!(
|
||||
"model_catalog_json = {}",
|
||||
serde_json::to_string(&catalog_path)?
|
||||
);
|
||||
std::fs::write(
|
||||
config_path,
|
||||
format!(
|
||||
r#"{config}
|
||||
r#"{catalog_config}
|
||||
{config}
|
||||
[features]
|
||||
step_model_switching = true
|
||||
default_mode_request_user_input = true
|
||||
plugins = true
|
||||
remote_plugin = false
|
||||
unified_exec = true
|
||||
@@ -787,7 +864,7 @@ enabled = true
|
||||
timeout(Duration::from_secs(10), mcp.initialize()).await??;
|
||||
let thread_request = mcp
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams {
|
||||
model: Some("mock-model".to_string()),
|
||||
model: Some("initial-model".to_string()),
|
||||
service_name: Some("codex_work_desktop".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
@@ -813,11 +890,43 @@ enabled = true
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
timeout(
|
||||
let turn_response = timeout(
|
||||
Duration::from_secs(10),
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_request)),
|
||||
)
|
||||
.await??;
|
||||
let turn_response: TurnStartResponse = to_response(turn_response)?;
|
||||
for model in if background {
|
||||
vec!["invoking-model", "initial-model"]
|
||||
} else {
|
||||
vec!["invoking-model"]
|
||||
} {
|
||||
let request = timeout(
|
||||
Duration::from_secs(10),
|
||||
mcp.read_stream_until_request_message(),
|
||||
)
|
||||
.await??;
|
||||
let ServerRequest::ToolRequestUserInput { request_id, .. } = request else {
|
||||
anyhow::bail!("expected request_user_input, received {request:?}");
|
||||
};
|
||||
let response: TurnSettingsUpdateResponse = mcp
|
||||
.request(|request_id| ClientRequest::TurnSettingsUpdate {
|
||||
request_id,
|
||||
params: TurnSettingsUpdateParams {
|
||||
thread_id: thread_id.clone(),
|
||||
turn_id: turn_response.turn.id.clone(),
|
||||
model: Some(model.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
assert_eq!(response.status, TurnSettingsUpdateStatus::Applied);
|
||||
mcp.send_response(
|
||||
request_id,
|
||||
json!({"answers": {"continue": {"answers": ["Yes"]}}}),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let completed_turn = timeout(
|
||||
Duration::from_secs(10),
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
@@ -830,17 +939,7 @@ enabled = true
|
||||
.expect("completed turn id");
|
||||
|
||||
if background {
|
||||
let model_request_bodies = server
|
||||
.received_requests()
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.filter(|request| request.url.path().ends_with("/responses"))
|
||||
.map(|request| serde_json::from_slice::<Value>(&request.body))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let request_text = serde_json::to_string(&model_request_bodies)?;
|
||||
assert!(request_text.contains("Process running with session ID "));
|
||||
assert!(!request_text.contains("Process exited with code 0"));
|
||||
std::fs::write(release_path, "release")?;
|
||||
}
|
||||
|
||||
for measurement_name in ["findings", "files_scanned"] {
|
||||
@@ -859,12 +958,16 @@ enabled = true
|
||||
.await?;
|
||||
assert_eq!(
|
||||
json!({
|
||||
"model_slug": command_event["event_params"]["model_slug"],
|
||||
"reasoning_effort": command_event["event_params"]["reasoning_effort"],
|
||||
"plugin_id": command_event["event_params"]["plugin_id"],
|
||||
"script_path": command_event["event_params"]["script_path"],
|
||||
"item_id": command_event["event_params"]["item_id"],
|
||||
"exit_code": command_event["event_params"]["exit_code"],
|
||||
}),
|
||||
json!({
|
||||
"model_slug": "invoking-model",
|
||||
"reasoning_effort": "high",
|
||||
"plugin_id": METRICS_PLUGIN_ID,
|
||||
"script_path": "scripts/run.sh",
|
||||
"item_id": call_id,
|
||||
@@ -952,10 +1055,12 @@ enabled = true
|
||||
"execution_id",
|
||||
"item_id",
|
||||
"measurement_name",
|
||||
"model_slug",
|
||||
"number_value",
|
||||
"operation",
|
||||
"originator",
|
||||
"plugin_id",
|
||||
"reasoning_effort",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
]
|
||||
@@ -963,6 +1068,8 @@ enabled = true
|
||||
assert_eq!(event_params["thread_id"], thread_id);
|
||||
assert_eq!(event_params["turn_id"], turn_id);
|
||||
assert_eq!(event_params["originator"], "codex_work_desktop");
|
||||
assert_eq!(event_params["model_slug"], "invoking-model");
|
||||
assert_eq!(event_params["reasoning_effort"], "high");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user