openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-18 17:18:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Track built-in control tool calls in analytics (#39510)

Browse Source 
## What changed

- Emit `codex_control_tool_call_event` for built-in control tools such as
  `request_user_input`, `update_plan`, `view_image`, and goal tools.
- Record correlation and timing metadata plus completed, failed, rejected, or
  interrupted outcomes, and include these calls in turn-level tool counts.
- Keep tool arguments out of control-tool analytics events.

## Testing

- Extend the app-server turn profile test to cover successful and failed
  control tools, event correlation, turn counts, and argument exclusion.

GitOrigin-RevId: c20f73dfb79654e31b4b5040eddb0e0d91cae10d
This commit is contained in:
jif
2026-08-19 15:55:38 +00:00
committed by copyberry
parent 6972c57c78
commit 992f5c681f
13 changed files with 343 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/analytics/src/client.rs
+6
View File
@@ -336,6 +336,12 @@ impl AnalyticsEventsClient {
));
}
pub fn track_control_tool_call(&self, input: crate::facts::ControlToolCallFact) {
self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::ControlToolCall(
input,
)));
}
pub fn track_guardian_review(
&self,
tracking: &GuardianReviewTrackContext,
codex-rs/analytics/src/events.rs
+14
View File
@@ -82,6 +82,7 @@ pub(crate) enum TrackEventRequest {
FileChange(CodexFileChangeEventRequest),
McpToolCall(CodexMcpToolCallEventRequest),
DynamicToolCall(CodexDynamicToolCallEventRequest),
ControlToolCall(CodexControlToolCallEventRequest),
CollabAgentToolCall(CodexCollabAgentToolCallEventRequest),
WebSearch(CodexWebSearchEventRequest),
ImageGeneration(CodexImageGenerationEventRequest),
@@ -827,6 +828,19 @@ pub(crate) struct CodexDynamicToolCallEventRequest {
pub(crate) event_params: CodexDynamicToolCallEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexControlToolCallEventParams {
#[serde(flatten)]
pub(crate) base: CodexToolItemEventBase,
pub(crate) success: bool,
}
#[derive(Serialize)]
pub(crate) struct CodexControlToolCallEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: CodexControlToolCallEventParams,
}
#[derive(Serialize)]
pub(crate) struct CodexCollabAgentToolCallEventParams {
#[serde(flatten)]
codex-rs/analytics/src/facts.rs
+21
View File
@@ -107,6 +107,26 @@ pub enum CodeModeToolCallStatus {
Interrupted,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ControlToolCallFact {
pub thread_id: String,
pub turn_id: String,
pub call_id: String,
pub cell_id: Option<String>,
pub tool_name: String,
pub started_at_ms: u64,
pub completed_at_ms: u64,
pub status: ControlToolCallStatus,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ControlToolCallStatus {
Completed,
Failed,
Rejected,
Interrupted,
}
pub fn build_track_events_context(
model_slug: String,
thread_id: String,
@@ -521,6 +541,7 @@ pub(crate) enum AnalyticsFact {
pub(crate) enum CustomAnalyticsFact {
ArtifactOperation(ArtifactOperationInput),
CodeModeToolCall(CodeModeToolCallFact),
ControlToolCall(ControlToolCallFact),
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
codex-rs/analytics/src/lib.rs
+2
View File
@@ -38,6 +38,8 @@ pub use facts::CompactionReason;
pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
pub use facts::ControlToolCallFact;
pub use facts::ControlToolCallStatus;
pub use facts::ExternalAgentConfigImportCompletedInput;
pub use facts::ExternalAgentConfigImportFailureInput;
pub use facts::GoalEventKind;
codex-rs/analytics/src/reducer.rs
+85
View File
@@ -11,6 +11,8 @@ use crate::events::CodexCollabAgentToolCallEventRequest;
use crate::events::CodexCommandExecutionEventParams;
use crate::events::CodexCommandExecutionEventRequest;
use crate::events::CodexCompactionEventRequest;
use crate::events::CodexControlToolCallEventParams;
use crate::events::CodexControlToolCallEventRequest;
use crate::events::CodexDynamicToolCallEventParams;
use crate::events::CodexDynamicToolCallEventRequest;
use crate::events::CodexFileChangeEventParams;
@@ -82,6 +84,8 @@ use crate::facts::CodeModeToolCallFact;
use crate::facts::CodeModeToolCallStatus;
use crate::facts::CodexCompactionEvent;
use crate::facts::CodexGoalEvent;
use crate::facts::ControlToolCallFact;
use crate::facts::ControlToolCallStatus;
use crate::facts::CustomAnalyticsFact;
use crate::facts::ExternalAgentConfigImportCompletedInput;
use crate::facts::ExternalAgentConfigImportFailureInput;
@@ -589,6 +593,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::CodeModeToolCall(input) => {
self.ingest_code_mode_tool_call(input, out);
}
CustomAnalyticsFact::ControlToolCall(input) => {
self.ingest_control_tool_call(input, out);
}
CustomAnalyticsFact::SubAgentThreadStarted(input) => {
self.ingest_subagent_thread_started(input, out);
}
@@ -833,6 +840,83 @@ impl AnalyticsReducer {
}
}
fn ingest_control_tool_call(
&mut self,
input: ControlToolCallFact,
out: &mut Vec<TrackEventRequest>,
) {
let ControlToolCallFact {
thread_id,
turn_id,
call_id,
cell_id,
tool_name,
started_at_ms,
completed_at_ms,
status,
} = input;
let drop_site = AnalyticsDropSite {
event_name: "control tool",
thread_id: &thread_id,
turn_id: Some(&turn_id),
review_id: None,
item_id: Some(&call_id),
};
let Some((connection_state, thread_state, thread_metadata)) =
self.thread_context_or_warn(drop_site)
else {
return;
};
let mut base = tool_item_base(
&thread_id,
&turn_id,
call_id,
tool_name,
ToolItemOutcome {
terminal_status: match status {
ControlToolCallStatus::Completed => ToolItemTerminalStatus::Completed,
ControlToolCallStatus::Failed => ToolItemTerminalStatus::Failed,
ControlToolCallStatus::Rejected => ToolItemTerminalStatus::Rejected,
ControlToolCallStatus::Interrupted => ToolItemTerminalStatus::Interrupted,
},
failure_kind: match status {
ControlToolCallStatus::Failed => Some(ToolItemFailureKind::ToolError),
ControlToolCallStatus::Rejected => Some(ToolItemFailureKind::PolicyForbidden),
ControlToolCallStatus::Completed | ControlToolCallStatus::Interrupted => None,
},
execution_duration_ms: observed_duration_ms(started_at_ms, completed_at_ms),
},
ToolItemContext {
started_at_ms,
completed_at_ms,
connection_state,
thread_state,
thread_metadata,
review_summary: None,
},
);
base.cell_id = cell_id;
let event = TrackEventRequest::ControlToolCall(CodexControlToolCallEventRequest {
event_type: "codex_control_tool_call_event",
event_params: CodexControlToolCallEventParams {
base,
success: status == ControlToolCallStatus::Completed,
},
});
self.turns
.entry(turn_id.clone())
.or_default()
.tool_counts
.total += 1;
self.record_tool_event(
&thread_id,
&turn_id,
event,
ToolEventEmission::AwaitResponse,
out,
);
}
fn ingest_sampling_response_completed(
&mut self,
thread_id: String,
@@ -2307,6 +2391,7 @@ fn tool_event_base_mut(event: &mut TrackEventRequest) -> Option<&mut CodexToolIt
TrackEventRequest::FileChange(event) => Some(&mut event.event_params.base),
TrackEventRequest::McpToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::DynamicToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::ControlToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::CollabAgentToolCall(event) => Some(&mut event.event_params.base),
TrackEventRequest::WebSearch(event) => Some(&mut event.event_params.base),
TrackEventRequest::ImageGeneration(event) => Some(&mut event.event_params.base),
codex-rs/app-server/tests/suite/v2/turn_start.rs
+113 -6
View File
@@ -95,6 +95,7 @@ use wiremock::ResponseTemplate;
use super::analytics::mount_analytics_capture;
use super::analytics::wait_for_analytics_event;
use super::analytics::wait_for_matching_analytics_event;
#[cfg(windows)]
const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25);
@@ -1109,18 +1110,50 @@ async fn code_mode_exec_emits_correlated_production_analytics() -> Result<()> {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()> {
let responses = vec![
let control_tools = [
("call1", "request_user_input", json!({})),
(
"plan-call",
"update_plan",
json!({"plan": [{"step": "PRIVATE_PLAN", "status": "in_progress"}]}),
),
(
"image-call",
"view_image",
json!({"path": "PRIVATE_IMAGE_PATH.png"}),
),
("get-goal-call", "get_goal", json!({})),
(
"create-goal-call",
"create_goal",
json!({"objective": "PRIVATE_GOAL"}),
),
(
"update-goal-call",
"update_goal",
json!({"status": "complete"}),
),
];
let mut responses = vec![
create_request_user_input_sse_response("call1")?,
create_final_assistant_message_sse_response("Done")?,
];
responses.extend(control_tools.iter().skip(1).map(|(call_id, tool, args)| {
let response_id = format!("resp-{call_id}");
responses::sse(vec![
responses::ev_response_created(&response_id),
responses::ev_function_call(call_id, tool, &args.to_string()),
responses::ev_completed(&response_id),
])
}));
responses.push(create_final_assistant_message_sse_response("Done")?);
let server = create_mock_responses_server_sequence(responses).await;
let codex_home = TempDir::new()?;
write_mock_responses_config_toml_with_chatgpt_base_url(
codex_home.path(),
&server.uri(),
&server.uri(),
)?;
MockResponsesConfig::new(&server.uri())
.enable_feature(Feature::Goals)
.with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
.write(codex_home.path())?;
mount_analytics_capture(&server, codex_home.path()).await?;
let mut mcp = TestAppServer::builder()
@@ -1193,16 +1226,90 @@ async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()
.is_some_and(|duration| duration > 0),
"samplingRequestCount": params["sampling_request_count"],
"samplingRetryCount": params["sampling_retry_count"],
"totalToolCalls": params["total_tool_call_count"],
"dynamicToolCalls": params["dynamic_tool_call_count"],
"status": params["status"],
}),
json!({
"toolBlockingIsPositive": true,
"samplingRequestCount": 2,
"samplingRetryCount": 0,
"totalToolCalls": 1,
"dynamicToolCalls": 0,
"status": "completed",
})
);
let second_turn = mcp
.start_turn_and_wait_for_completion(TurnStartParams {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "run the other control tools".to_string(),
text_elements: Vec::new(),
}],
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Default,
settings: Settings {
model: "mock-model".to_string(),
reasoning_effort: Some(ReasoningEffort::Medium),
developer_instructions: None,
},
}),
..Default::default()
})
.await?;
for (call_id, tool_name, _) in control_tools {
let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
event["event_type"] == "codex_control_tool_call_event"
&& event["event_params"]["item_id"] == call_id
})
.await?;
let success = tool_name != "view_image";
assert_eq!(
json!({
"tool": event["event_params"]["tool_name"],
"success": event["event_params"]["success"],
"status": event["event_params"]["terminal_status"],
"hasOrigin": event["event_params"]["originating_response_id"]
.as_str()
.is_some(),
}),
json!({
"tool": tool_name,
"success": success,
"status": if success { "completed" } else { "failed" },
"hasOrigin": true,
})
);
let serialized = event.to_string();
assert!(
![
"PRIVATE_PLAN",
"PRIVATE_IMAGE_PATH",
"PRIVATE_GOAL",
"Proceed with the plan?"
]
.iter()
.any(|private_input| serialized.contains(private_input)),
"tool-call analytics must not contain private tool arguments: {serialized}"
);
}
let second_turn_event =
wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
event["event_type"] == "codex_turn_event"
&& event["event_params"]["turn_id"] == second_turn.turn.id
})
.await?;
assert_eq!(
json!({
"total": second_turn_event["event_params"]["total_tool_call_count"],
"dynamic": second_turn_event["event_params"]["dynamic_tool_call_count"],
}),
json!({"total": 5, "dynamic": 0})
);
Ok(())
}
codex-rs/core/src/tools/control_tool_analytics.rs
+52
View File
@@ -0,0 +1,52 @@
use crate::tools::context::ToolCallSource;
use crate::tools::context::ToolInvocation;
use codex_analytics::ControlToolCallFact;
use codex_analytics::ControlToolCallStatus;
pub(crate) struct ControlToolCallGuard<'a> {
invocation: &'a ToolInvocation,
started_at_ms: u64,
completed_at_ms: Option<u64>,
status: ControlToolCallStatus,
}
impl<'a> ControlToolCallGuard<'a> {
pub(crate) fn new(invocation: &'a ToolInvocation) -> Self {
Self {
invocation,
started_at_ms: codex_analytics::now_unix_millis(),
completed_at_ms: None,
status: ControlToolCallStatus::Interrupted,
}
}
pub(crate) fn finish(&mut self, status: ControlToolCallStatus) {
self.completed_at_ms = Some(codex_analytics::now_unix_millis());
self.status = status;
}
}
impl Drop for ControlToolCallGuard<'_> {
fn drop(&mut self) {
let invocation = self.invocation;
invocation
.session
.services
.analytics_events_client
.track_control_tool_call(ControlToolCallFact {
thread_id: invocation.session.thread_id.to_string(),
turn_id: invocation.turn.sub_id.clone(),
call_id: invocation.call_id.clone(),
cell_id: match &invocation.source {
ToolCallSource::CodeMode { cell_id, .. } => Some(cell_id.clone()),
ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => None,
},
tool_name: invocation.tool_name.name.clone(),
started_at_ms: self.started_at_ms,
completed_at_ms: self
.completed_at_ms
.unwrap_or_else(codex_analytics::now_unix_millis),
status: self.status,
});
}
}
codex-rs/core/src/tools/handlers/extension_tools.rs
+9
View File
@@ -61,6 +61,15 @@ impl ToolExecutor<ToolInvocation> for ExtensionToolAdapter {
}
impl CoreToolRuntime for ExtensionToolAdapter {
fn is_builtin_control_tool(&self) -> bool {
let tool_name = self.0.tool_name();
tool_name.is_default_namespace()
&& matches!(
tool_name.name.as_str(),
"get_goal" | "create_goal" | "update_goal"
)
}
fn matches_kind(&self, payload: &ToolPayload) -> bool {
match payload {
ToolPayload::Function { .. } => true,
codex-rs/core/src/tools/handlers/plan.rs
+5 -1
View File
@@ -96,7 +96,11 @@ impl PlanHandler {
}
}
impl CoreToolRuntime for PlanHandler {}
impl CoreToolRuntime for PlanHandler {
fn is_builtin_control_tool(&self) -> bool {
true
}
}
fn parse_update_plan_arguments(arguments: &str) -> Result<UpdatePlanArgs, FunctionCallError> {
serde_json::from_str::<UpdatePlanArgs>(arguments).map_err(|e| {
codex-rs/core/src/tools/handlers/request_user_input.rs
+5 -1
View File
@@ -98,7 +98,11 @@ impl RequestUserInputHandler {
}
}
impl CoreToolRuntime for RequestUserInputHandler {}
impl CoreToolRuntime for RequestUserInputHandler {
fn is_builtin_control_tool(&self) -> bool {
true
}
}
#[cfg(test)]
#[path = "request_user_input_tests.rs"]
codex-rs/core/src/tools/handlers/view_image.rs
+5 -1
View File
@@ -208,7 +208,11 @@ impl ViewImageHandler {
}
}
impl CoreToolRuntime for ViewImageHandler {}
impl CoreToolRuntime for ViewImageHandler {
fn is_builtin_control_tool(&self) -> bool {
true
}
}
pub struct ViewImageOutput {
image_url: String,
codex-rs/core/src/tools/mod.rs
+1
View File
@@ -1,6 +1,7 @@
mod approvals;
pub(crate) mod code_mode;
pub(crate) mod context;
mod control_tool_analytics;
pub(crate) mod events;
mod executed_tool_calls;
pub(crate) mod handlers;
codex-rs/core/src/tools/registry.rs
+25
View File
@@ -19,6 +19,7 @@ use crate::tools::context::FunctionToolOutput;
use crate::tools::context::ToolInvocation;
use crate::tools::context::ToolOutput;
use crate::tools::context::ToolPayload;
use crate::tools::control_tool_analytics::ControlToolCallGuard;
use crate::tools::flat_tool_name;
use crate::tools::handlers::multi_agents_spec::MULTI_AGENT_V1_NAMESPACE;
use crate::tools::hook_names::HookToolName;
@@ -27,6 +28,7 @@ use crate::tools::lifecycle::notify_tool_start;
use crate::tools::router::tool_log_payload;
use crate::tools::tool_dispatch_trace::ToolDispatchTrace;
use crate::util::error_or_panic;
use codex_analytics::ControlToolCallStatus;
use codex_extension_api::ToolCallOutcome;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseInputItem;
@@ -51,6 +53,11 @@ pub use codex_tools::ToolExposure;
/// Implementers provide the shared `ToolExecutor` behavior plus optional
/// core-owned metadata for hooks, telemetry, tool search, and argument diffs.
pub(crate) trait CoreToolRuntime: ToolExecutor<ToolInvocation> {
/// Whether this built-in control tool needs a structured tool-call event.
fn is_builtin_control_tool(&self) -> bool {
false
}
/// Returns a shared spec when both the spec and search metadata are immutable.
fn immutable_spec(&self) -> Option<&Arc<ToolSpec>> {
None
@@ -577,6 +584,10 @@ impl ToolRegistry {
.await
{
PreToolUseHookResult::Blocked(message) => {
if tool.is_builtin_control_tool() {
let mut analytics = ControlToolCallGuard::new(&invocation);
analytics.finish(ControlToolCallStatus::Rejected);
}
let err = FunctionCallError::RespondToModel(message);
dispatch_trace.record_failed(&err);
notify_tool_finish_if_unclaimed(
@@ -594,6 +605,10 @@ impl ToolRegistry {
invocation = updated_invocation;
}
Err(err) => {
if tool.is_builtin_control_tool() {
let mut analytics = ControlToolCallGuard::new(&invocation);
analytics.finish(ControlToolCallStatus::Failed);
}
dispatch_trace.record_failed(&err);
notify_tool_finish_if_unclaimed(
&invocation,
@@ -613,6 +628,9 @@ impl ToolRegistry {
}
notify_tool_start(&invocation).await;
let mut control_tool_analytics = tool
.is_builtin_control_tool()
.then(|| ControlToolCallGuard::new(&invocation));
if let Some(command) = shell_script_for_invocation(&invocation) {
let parsed = parse_shell_script(&command);
@@ -663,6 +681,13 @@ impl ToolRegistry {
Ok((_, success)) => *success,
Err(_) => false,
};
if let Some(analytics) = control_tool_analytics.as_mut() {
analytics.finish(if success {
ControlToolCallStatus::Completed
} else {
ControlToolCallStatus::Failed
});
}
emit_metric_for_tool_read(&invocation, success);
let post_tool_use_payload = if success {
let guard = response_cell.lock().await;