diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 6791aee084..00ec2d8d4c 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -336,6 +336,12 @@ impl AnalyticsEventsClient { )); } + pub fn track_control_tool_call(&self, input: crate::facts::ControlToolCallFact) { + self.record_fact(AnalyticsFact::Custom(CustomAnalyticsFact::ControlToolCall( + input, + ))); + } + pub fn track_guardian_review( &self, tracking: &GuardianReviewTrackContext, diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index af431721eb..88b6870619 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -82,6 +82,7 @@ pub(crate) enum TrackEventRequest { FileChange(CodexFileChangeEventRequest), McpToolCall(CodexMcpToolCallEventRequest), DynamicToolCall(CodexDynamicToolCallEventRequest), + ControlToolCall(CodexControlToolCallEventRequest), CollabAgentToolCall(CodexCollabAgentToolCallEventRequest), WebSearch(CodexWebSearchEventRequest), ImageGeneration(CodexImageGenerationEventRequest), @@ -827,6 +828,19 @@ pub(crate) struct CodexDynamicToolCallEventRequest { pub(crate) event_params: CodexDynamicToolCallEventParams, } +#[derive(Serialize)] +pub(crate) struct CodexControlToolCallEventParams { + #[serde(flatten)] + pub(crate) base: CodexToolItemEventBase, + pub(crate) success: bool, +} + +#[derive(Serialize)] +pub(crate) struct CodexControlToolCallEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexControlToolCallEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexCollabAgentToolCallEventParams { #[serde(flatten)] diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index f43acd9b06..3ca5b3327c 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -107,6 +107,26 @@ pub enum CodeModeToolCallStatus { Interrupted, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ControlToolCallFact { + pub thread_id: String, + pub turn_id: String, + pub call_id: String, + pub cell_id: Option, + pub tool_name: String, + pub started_at_ms: u64, + pub completed_at_ms: u64, + pub status: ControlToolCallStatus, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ControlToolCallStatus { + Completed, + Failed, + Rejected, + Interrupted, +} + pub fn build_track_events_context( model_slug: String, thread_id: String, @@ -521,6 +541,7 @@ pub(crate) enum AnalyticsFact { pub(crate) enum CustomAnalyticsFact { ArtifactOperation(ArtifactOperationInput), CodeModeToolCall(CodeModeToolCallFact), + ControlToolCall(ControlToolCallFact), SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), Goal(Box), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 7e7142abd3..942bfd2b95 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -38,6 +38,8 @@ pub use facts::CompactionReason; pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; +pub use facts::ControlToolCallFact; +pub use facts::ControlToolCallStatus; pub use facts::ExternalAgentConfigImportCompletedInput; pub use facts::ExternalAgentConfigImportFailureInput; pub use facts::GoalEventKind; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index c7be5aab88..974658ec20 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -11,6 +11,8 @@ use crate::events::CodexCollabAgentToolCallEventRequest; use crate::events::CodexCommandExecutionEventParams; use crate::events::CodexCommandExecutionEventRequest; use crate::events::CodexCompactionEventRequest; +use crate::events::CodexControlToolCallEventParams; +use crate::events::CodexControlToolCallEventRequest; use crate::events::CodexDynamicToolCallEventParams; use crate::events::CodexDynamicToolCallEventRequest; use crate::events::CodexFileChangeEventParams; @@ -82,6 +84,8 @@ use crate::facts::CodeModeToolCallFact; use crate::facts::CodeModeToolCallStatus; use crate::facts::CodexCompactionEvent; use crate::facts::CodexGoalEvent; +use crate::facts::ControlToolCallFact; +use crate::facts::ControlToolCallStatus; use crate::facts::CustomAnalyticsFact; use crate::facts::ExternalAgentConfigImportCompletedInput; use crate::facts::ExternalAgentConfigImportFailureInput; @@ -589,6 +593,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::CodeModeToolCall(input) => { self.ingest_code_mode_tool_call(input, out); } + CustomAnalyticsFact::ControlToolCall(input) => { + self.ingest_control_tool_call(input, out); + } CustomAnalyticsFact::SubAgentThreadStarted(input) => { self.ingest_subagent_thread_started(input, out); } @@ -833,6 +840,83 @@ impl AnalyticsReducer { } } + fn ingest_control_tool_call( + &mut self, + input: ControlToolCallFact, + out: &mut Vec, + ) { + let ControlToolCallFact { + thread_id, + turn_id, + call_id, + cell_id, + tool_name, + started_at_ms, + completed_at_ms, + status, + } = input; + let drop_site = AnalyticsDropSite { + event_name: "control tool", + thread_id: &thread_id, + turn_id: Some(&turn_id), + review_id: None, + item_id: Some(&call_id), + }; + let Some((connection_state, thread_state, thread_metadata)) = + self.thread_context_or_warn(drop_site) + else { + return; + }; + let mut base = tool_item_base( + &thread_id, + &turn_id, + call_id, + tool_name, + ToolItemOutcome { + terminal_status: match status { + ControlToolCallStatus::Completed => ToolItemTerminalStatus::Completed, + ControlToolCallStatus::Failed => ToolItemTerminalStatus::Failed, + ControlToolCallStatus::Rejected => ToolItemTerminalStatus::Rejected, + ControlToolCallStatus::Interrupted => ToolItemTerminalStatus::Interrupted, + }, + failure_kind: match status { + ControlToolCallStatus::Failed => Some(ToolItemFailureKind::ToolError), + ControlToolCallStatus::Rejected => Some(ToolItemFailureKind::PolicyForbidden), + ControlToolCallStatus::Completed | ControlToolCallStatus::Interrupted => None, + }, + execution_duration_ms: observed_duration_ms(started_at_ms, completed_at_ms), + }, + ToolItemContext { + started_at_ms, + completed_at_ms, + connection_state, + thread_state, + thread_metadata, + review_summary: None, + }, + ); + base.cell_id = cell_id; + let event = TrackEventRequest::ControlToolCall(CodexControlToolCallEventRequest { + event_type: "codex_control_tool_call_event", + event_params: CodexControlToolCallEventParams { + base, + success: status == ControlToolCallStatus::Completed, + }, + }); + self.turns + .entry(turn_id.clone()) + .or_default() + .tool_counts + .total += 1; + self.record_tool_event( + &thread_id, + &turn_id, + event, + ToolEventEmission::AwaitResponse, + out, + ); + } + fn ingest_sampling_response_completed( &mut self, thread_id: String, @@ -2307,6 +2391,7 @@ fn tool_event_base_mut(event: &mut TrackEventRequest) -> Option<&mut CodexToolIt TrackEventRequest::FileChange(event) => Some(&mut event.event_params.base), TrackEventRequest::McpToolCall(event) => Some(&mut event.event_params.base), TrackEventRequest::DynamicToolCall(event) => Some(&mut event.event_params.base), + TrackEventRequest::ControlToolCall(event) => Some(&mut event.event_params.base), TrackEventRequest::CollabAgentToolCall(event) => Some(&mut event.event_params.base), TrackEventRequest::WebSearch(event) => Some(&mut event.event_params.base), TrackEventRequest::ImageGeneration(event) => Some(&mut event.event_params.base), diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 0e6f464228..cd107bf668 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -95,6 +95,7 @@ use wiremock::ResponseTemplate; use super::analytics::mount_analytics_capture; use super::analytics::wait_for_analytics_event; +use super::analytics::wait_for_matching_analytics_event; #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); @@ -1109,18 +1110,50 @@ async fn code_mode_exec_emits_correlated_production_analytics() -> Result<()> { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<()> { - let responses = vec![ + let control_tools = [ + ("call1", "request_user_input", json!({})), + ( + "plan-call", + "update_plan", + json!({"plan": [{"step": "PRIVATE_PLAN", "status": "in_progress"}]}), + ), + ( + "image-call", + "view_image", + json!({"path": "PRIVATE_IMAGE_PATH.png"}), + ), + ("get-goal-call", "get_goal", json!({})), + ( + "create-goal-call", + "create_goal", + json!({"objective": "PRIVATE_GOAL"}), + ), + ( + "update-goal-call", + "update_goal", + json!({"status": "complete"}), + ), + ]; + let mut responses = vec![ create_request_user_input_sse_response("call1")?, create_final_assistant_message_sse_response("Done")?, ]; + responses.extend(control_tools.iter().skip(1).map(|(call_id, tool, args)| { + let response_id = format!("resp-{call_id}"); + responses::sse(vec![ + responses::ev_response_created(&response_id), + responses::ev_function_call(call_id, tool, &args.to_string()), + responses::ev_completed(&response_id), + ]) + })); + responses.push(create_final_assistant_message_sse_response("Done")?); let server = create_mock_responses_server_sequence(responses).await; let codex_home = TempDir::new()?; - write_mock_responses_config_toml_with_chatgpt_base_url( - codex_home.path(), - &server.uri(), - &server.uri(), - )?; + MockResponsesConfig::new(&server.uri()) + .enable_feature(Feature::Goals) + .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri())) + .write(codex_home.path())?; mount_analytics_capture(&server, codex_home.path()).await?; let mut mcp = TestAppServer::builder() @@ -1193,16 +1226,90 @@ async fn turn_profile_tracks_blocking_tool_and_follow_up_sampling() -> Result<() .is_some_and(|duration| duration > 0), "samplingRequestCount": params["sampling_request_count"], "samplingRetryCount": params["sampling_retry_count"], + "totalToolCalls": params["total_tool_call_count"], + "dynamicToolCalls": params["dynamic_tool_call_count"], "status": params["status"], }), json!({ "toolBlockingIsPositive": true, "samplingRequestCount": 2, "samplingRetryCount": 0, + "totalToolCalls": 1, + "dynamicToolCalls": 0, "status": "completed", }) ); + let second_turn = mcp + .start_turn_and_wait_for_completion(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Text { + text: "run the other control tools".to_string(), + text_elements: Vec::new(), + }], + collaboration_mode: Some(CollaborationMode { + mode: ModeKind::Default, + settings: Settings { + model: "mock-model".to_string(), + reasoning_effort: Some(ReasoningEffort::Medium), + developer_instructions: None, + }, + }), + ..Default::default() + }) + .await?; + + for (call_id, tool_name, _) in control_tools { + let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| { + event["event_type"] == "codex_control_tool_call_event" + && event["event_params"]["item_id"] == call_id + }) + .await?; + let success = tool_name != "view_image"; + assert_eq!( + json!({ + "tool": event["event_params"]["tool_name"], + "success": event["event_params"]["success"], + "status": event["event_params"]["terminal_status"], + "hasOrigin": event["event_params"]["originating_response_id"] + .as_str() + .is_some(), + }), + json!({ + "tool": tool_name, + "success": success, + "status": if success { "completed" } else { "failed" }, + "hasOrigin": true, + }) + ); + let serialized = event.to_string(); + assert!( + ![ + "PRIVATE_PLAN", + "PRIVATE_IMAGE_PATH", + "PRIVATE_GOAL", + "Proceed with the plan?" + ] + .iter() + .any(|private_input| serialized.contains(private_input)), + "tool-call analytics must not contain private tool arguments: {serialized}" + ); + } + + let second_turn_event = + wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| { + event["event_type"] == "codex_turn_event" + && event["event_params"]["turn_id"] == second_turn.turn.id + }) + .await?; + assert_eq!( + json!({ + "total": second_turn_event["event_params"]["total_tool_call_count"], + "dynamic": second_turn_event["event_params"]["dynamic_tool_call_count"], + }), + json!({"total": 5, "dynamic": 0}) + ); + Ok(()) } diff --git a/codex-rs/core/src/tools/control_tool_analytics.rs b/codex-rs/core/src/tools/control_tool_analytics.rs new file mode 100644 index 0000000000..46719201e6 --- /dev/null +++ b/codex-rs/core/src/tools/control_tool_analytics.rs @@ -0,0 +1,52 @@ +use crate::tools::context::ToolCallSource; +use crate::tools::context::ToolInvocation; +use codex_analytics::ControlToolCallFact; +use codex_analytics::ControlToolCallStatus; + +pub(crate) struct ControlToolCallGuard<'a> { + invocation: &'a ToolInvocation, + started_at_ms: u64, + completed_at_ms: Option, + status: ControlToolCallStatus, +} + +impl<'a> ControlToolCallGuard<'a> { + pub(crate) fn new(invocation: &'a ToolInvocation) -> Self { + Self { + invocation, + started_at_ms: codex_analytics::now_unix_millis(), + completed_at_ms: None, + status: ControlToolCallStatus::Interrupted, + } + } + + pub(crate) fn finish(&mut self, status: ControlToolCallStatus) { + self.completed_at_ms = Some(codex_analytics::now_unix_millis()); + self.status = status; + } +} + +impl Drop for ControlToolCallGuard<'_> { + fn drop(&mut self) { + let invocation = self.invocation; + invocation + .session + .services + .analytics_events_client + .track_control_tool_call(ControlToolCallFact { + thread_id: invocation.session.thread_id.to_string(), + turn_id: invocation.turn.sub_id.clone(), + call_id: invocation.call_id.clone(), + cell_id: match &invocation.source { + ToolCallSource::CodeMode { cell_id, .. } => Some(cell_id.clone()), + ToolCallSource::Direct | ToolCallSource::DirectPlaintextMessage => None, + }, + tool_name: invocation.tool_name.name.clone(), + started_at_ms: self.started_at_ms, + completed_at_ms: self + .completed_at_ms + .unwrap_or_else(codex_analytics::now_unix_millis), + status: self.status, + }); + } +} diff --git a/codex-rs/core/src/tools/handlers/extension_tools.rs b/codex-rs/core/src/tools/handlers/extension_tools.rs index 87dc39563d..57b6e51e95 100644 --- a/codex-rs/core/src/tools/handlers/extension_tools.rs +++ b/codex-rs/core/src/tools/handlers/extension_tools.rs @@ -61,6 +61,15 @@ impl ToolExecutor for ExtensionToolAdapter { } impl CoreToolRuntime for ExtensionToolAdapter { + fn is_builtin_control_tool(&self) -> bool { + let tool_name = self.0.tool_name(); + tool_name.is_default_namespace() + && matches!( + tool_name.name.as_str(), + "get_goal" | "create_goal" | "update_goal" + ) + } + fn matches_kind(&self, payload: &ToolPayload) -> bool { match payload { ToolPayload::Function { .. } => true, diff --git a/codex-rs/core/src/tools/handlers/plan.rs b/codex-rs/core/src/tools/handlers/plan.rs index 33b91aa512..d962314752 100644 --- a/codex-rs/core/src/tools/handlers/plan.rs +++ b/codex-rs/core/src/tools/handlers/plan.rs @@ -96,7 +96,11 @@ impl PlanHandler { } } -impl CoreToolRuntime for PlanHandler {} +impl CoreToolRuntime for PlanHandler { + fn is_builtin_control_tool(&self) -> bool { + true + } +} fn parse_update_plan_arguments(arguments: &str) -> Result { serde_json::from_str::(arguments).map_err(|e| { diff --git a/codex-rs/core/src/tools/handlers/request_user_input.rs b/codex-rs/core/src/tools/handlers/request_user_input.rs index 92e36fc668..f586f7afd4 100644 --- a/codex-rs/core/src/tools/handlers/request_user_input.rs +++ b/codex-rs/core/src/tools/handlers/request_user_input.rs @@ -98,7 +98,11 @@ impl RequestUserInputHandler { } } -impl CoreToolRuntime for RequestUserInputHandler {} +impl CoreToolRuntime for RequestUserInputHandler { + fn is_builtin_control_tool(&self) -> bool { + true + } +} #[cfg(test)] #[path = "request_user_input_tests.rs"] diff --git a/codex-rs/core/src/tools/handlers/view_image.rs b/codex-rs/core/src/tools/handlers/view_image.rs index 5e6bf3b0b4..ee8684350e 100644 --- a/codex-rs/core/src/tools/handlers/view_image.rs +++ b/codex-rs/core/src/tools/handlers/view_image.rs @@ -208,7 +208,11 @@ impl ViewImageHandler { } } -impl CoreToolRuntime for ViewImageHandler {} +impl CoreToolRuntime for ViewImageHandler { + fn is_builtin_control_tool(&self) -> bool { + true + } +} pub struct ViewImageOutput { image_url: String, diff --git a/codex-rs/core/src/tools/mod.rs b/codex-rs/core/src/tools/mod.rs index 3ca459a83e..5c29628079 100644 --- a/codex-rs/core/src/tools/mod.rs +++ b/codex-rs/core/src/tools/mod.rs @@ -1,6 +1,7 @@ mod approvals; pub(crate) mod code_mode; pub(crate) mod context; +mod control_tool_analytics; pub(crate) mod events; mod executed_tool_calls; pub(crate) mod handlers; diff --git a/codex-rs/core/src/tools/registry.rs b/codex-rs/core/src/tools/registry.rs index c02f3dec74..e7c416d077 100644 --- a/codex-rs/core/src/tools/registry.rs +++ b/codex-rs/core/src/tools/registry.rs @@ -19,6 +19,7 @@ use crate::tools::context::FunctionToolOutput; use crate::tools::context::ToolInvocation; use crate::tools::context::ToolOutput; use crate::tools::context::ToolPayload; +use crate::tools::control_tool_analytics::ControlToolCallGuard; use crate::tools::flat_tool_name; use crate::tools::handlers::multi_agents_spec::MULTI_AGENT_V1_NAMESPACE; use crate::tools::hook_names::HookToolName; @@ -27,6 +28,7 @@ use crate::tools::lifecycle::notify_tool_start; use crate::tools::router::tool_log_payload; use crate::tools::tool_dispatch_trace::ToolDispatchTrace; use crate::util::error_or_panic; +use codex_analytics::ControlToolCallStatus; use codex_extension_api::ToolCallOutcome; use codex_protocol::models::FunctionCallOutputPayload; use codex_protocol::models::ResponseInputItem; @@ -51,6 +53,11 @@ pub use codex_tools::ToolExposure; /// Implementers provide the shared `ToolExecutor` behavior plus optional /// core-owned metadata for hooks, telemetry, tool search, and argument diffs. pub(crate) trait CoreToolRuntime: ToolExecutor { + /// Whether this built-in control tool needs a structured tool-call event. + fn is_builtin_control_tool(&self) -> bool { + false + } + /// Returns a shared spec when both the spec and search metadata are immutable. fn immutable_spec(&self) -> Option<&Arc> { None @@ -577,6 +584,10 @@ impl ToolRegistry { .await { PreToolUseHookResult::Blocked(message) => { + if tool.is_builtin_control_tool() { + let mut analytics = ControlToolCallGuard::new(&invocation); + analytics.finish(ControlToolCallStatus::Rejected); + } let err = FunctionCallError::RespondToModel(message); dispatch_trace.record_failed(&err); notify_tool_finish_if_unclaimed( @@ -594,6 +605,10 @@ impl ToolRegistry { invocation = updated_invocation; } Err(err) => { + if tool.is_builtin_control_tool() { + let mut analytics = ControlToolCallGuard::new(&invocation); + analytics.finish(ControlToolCallStatus::Failed); + } dispatch_trace.record_failed(&err); notify_tool_finish_if_unclaimed( &invocation, @@ -613,6 +628,9 @@ impl ToolRegistry { } notify_tool_start(&invocation).await; + let mut control_tool_analytics = tool + .is_builtin_control_tool() + .then(|| ControlToolCallGuard::new(&invocation)); if let Some(command) = shell_script_for_invocation(&invocation) { let parsed = parse_shell_script(&command); @@ -663,6 +681,13 @@ impl ToolRegistry { Ok((_, success)) => *success, Err(_) => false, }; + if let Some(analytics) = control_tool_analytics.as_mut() { + analytics.finish(if success { + ControlToolCallStatus::Completed + } else { + ControlToolCallStatus::Failed + }); + } emit_metric_for_tool_read(&invocation, success); let post_tool_use_payload = if success { let guard = response_cell.lock().await;