From c51e7b373647cf99208a1f391d05806ac4c3c9d7 Mon Sep 17 00:00:00 2001 From: jwang-openai Date: Tue, 25 Aug 2026 22:16:07 +0000 Subject: [PATCH] Add plugin-attributed skill telemetry (#40724) ## What changed - Add `plugin_id`, `model_slug`, and `reasoning_effort` dimensions to `codex.skill.injected` metrics for explicit and implicit skill invocations. - Propagate plugin IDs from orchestrator skill metadata so resource-backed skill invocations can emit the same attribution. - Record `codex.skill.turn.duration_seconds` once per plugin used in a turn, tagged with the model, reasoning effort, and completed, aborted, or error status. GitOrigin-RevId: d249a9c4e85682b1106ac49d9e598535c6cbb06d --- codex-rs/core/src/skills.rs | 23 +++ codex-rs/ext/skills/src/extension.rs | 2 + codex-rs/ext/skills/src/lib.rs | 2 + .../ext/skills/src/provider/orchestrator.rs | 4 + codex-rs/ext/skills/src/telemetry.rs | 159 ++++++++++++++++++ codex-rs/ext/skills/src/tools/mod.rs | 45 +++++ 6 files changed, 235 insertions(+) create mode 100644 codex-rs/ext/skills/src/telemetry.rs diff --git a/codex-rs/core/src/skills.rs b/codex-rs/core/src/skills.rs index 4d35dd403a..152073f39b 100644 --- a/codex-rs/core/src/skills.rs +++ b/codex-rs/core/src/skills.rs @@ -13,6 +13,7 @@ use codex_protocol::protocol::SkillScope; use codex_skills::SkillMetadata; use codex_skills_extension::HostSkillsLoadInput; use codex_skills_extension::detect_implicit_skill_invocation; +use codex_skills_extension::record_plugin_turn_usage; use codex_utils_absolute_path::AbsolutePathBuf; use codex_utils_path_uri::PathUri; use codex_utils_plugins::PluginSkillRoot; @@ -44,9 +45,17 @@ pub(crate) fn emit_explicit_skill_invocations( .iter() .map(|skill| &skill.path_to_skills_md) .collect::>(); + let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str()); + let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing(); for skill in mentioned_skills { let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str()); + let plugin_id_tag = + sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed")); let status = if injected_skill_paths.contains(&skill.path_to_skills_md) { + record_plugin_turn_usage( + turn_context.extension_data.as_ref(), + skill.plugin_id.as_deref(), + ); "ok" } else { "error" @@ -58,6 +67,9 @@ pub(crate) fn emit_explicit_skill_invocations( ("status", status), ("skill", skill_name_tag.as_str()), ("invoke_type", "explicit"), + ("plugin_id", plugin_id_tag.as_str()), + ("model_slug", model_slug_tag.as_str()), + ("reasoning_effort", reasoning_effort.as_str()), ], ); } @@ -123,6 +135,14 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation( return; } let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str()); + let plugin_id_tag = + sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed")); + let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str()); + let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing(); + record_plugin_turn_usage( + turn_context.extension_data.as_ref(), + invocation.plugin_id.as_deref(), + ); for contributor in sess.services.extensions.skill_invocation_contributors() { contributor @@ -144,6 +164,9 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation( ("status", "ok"), ("skill", skill_name_tag.as_str()), ("invoke_type", "implicit"), + ("plugin_id", plugin_id_tag.as_str()), + ("model_slug", model_slug_tag.as_str()), + ("reasoning_effort", reasoning_effort.as_str()), ], ); sess.services diff --git a/codex-rs/ext/skills/src/extension.rs b/codex-rs/ext/skills/src/extension.rs index c77e26665a..99cefa6229 100644 --- a/codex-rs/ext/skills/src/extension.rs +++ b/codex-rs/ext/skills/src/extension.rs @@ -67,6 +67,7 @@ use crate::state::HostSkillsStepState; use crate::state::SkillsSessionState; use crate::state::SkillsThreadState; use crate::state::SkillsTurnState; +use crate::telemetry::SkillTelemetry; use crate::tools::SkillAnalytics; use crate::tools::SkillToolAuthority; use crate::tools::skill_tools; @@ -670,6 +671,7 @@ pub fn install_with_providers_and_metrics( shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)), }); registry.thread_lifecycle_contributor(extension.clone()); + registry.turn_lifecycle_contributor(Arc::new(SkillTelemetry)); registry.config_contributor(extension.clone()); registry.prompt_contributor(extension.clone()); registry.turn_input_contributor(extension.clone()); diff --git a/codex-rs/ext/skills/src/lib.rs b/codex-rs/ext/skills/src/lib.rs index 940aa9657c..b3fdc291b9 100644 --- a/codex-rs/ext/skills/src/lib.rs +++ b/codex-rs/ext/skills/src/lib.rs @@ -20,6 +20,7 @@ mod selection; mod shadow_selection_experiment; mod sources; mod state; +mod telemetry; mod tools; mod warnings; mod world_state; @@ -43,6 +44,7 @@ pub use provider::OrchestratorSkillProvider; pub use provider::SkillProvider; pub use sources::SkillProviderSource; pub use sources::SkillProviders; +pub use telemetry::record_plugin_turn_usage; /// Recognizes persisted explicit skill prompts without exposing their fragment implementation. pub fn is_skill_prompt_fragment(text: &str) -> bool { diff --git a/codex-rs/ext/skills/src/provider/orchestrator.rs b/codex-rs/ext/skills/src/provider/orchestrator.rs index 07643b5ca7..e984693620 100644 --- a/codex-rs/ext/skills/src/provider/orchestrator.rs +++ b/codex-rs/ext/skills/src/provider/orchestrator.rs @@ -276,6 +276,10 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option ) .with_display_path(uri) .with_alias_root(format!("skill://{namespace}")); + entry.plugin_id = meta + .get("plugin_id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned); entry.canonical_skill_id = meta .get("skill_id") .and_then(serde_json::Value::as_str) diff --git a/codex-rs/ext/skills/src/telemetry.rs b/codex-rs/ext/skills/src/telemetry.rs new file mode 100644 index 0000000000..070cbb9762 --- /dev/null +++ b/codex-rs/ext/skills/src/telemetry.rs @@ -0,0 +1,159 @@ +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::Weak; +use std::time::Instant; + +use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionFuture; +use codex_extension_api::TurnAbortInput; +use codex_extension_api::TurnErrorInput; +use codex_extension_api::TurnLifecycleContributor; +use codex_extension_api::TurnStartInput; +use codex_extension_api::TurnStopInput; +use codex_otel::sanitize_metric_tag_value; +use codex_protocol::openai_models::ModelInfo; + +use crate::state::SkillsSessionState; + +/// Attributes a successful host skill invocation to its turn's plugin latency metrics. +pub fn record_plugin_turn_usage(turn_store: &ExtensionData, plugin_id: Option<&str>) { + if let Some(turn) = turn_store.get::() { + turn.record_plugin(plugin_id); + } +} + +pub(crate) struct SkillTurnMetrics { + pub(crate) turn_id: String, + model_slug: String, + pub(crate) reasoning_effort: String, + started_at: Instant, + usage: Mutex>, +} + +#[derive(Default)] +struct TurnUsage { + plugins: HashSet, + failed: bool, +} + +impl SkillTurnMetrics { + pub(crate) fn record_plugin(&self, plugin_id: Option<&str>) { + if let Some(usage) = self + .usage + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + usage + .plugins + .insert(plugin_id.unwrap_or("unattributed").to_string()); + } + } + + fn finish(&self, session_store: &ExtensionData, status: &str) { + let Some(usage) = self + .usage + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + else { + return; + }; + let Some(metrics) = session_store + .get::() + .and_then(|state| state.extension_metrics.clone()) + else { + return; + }; + let status = if status == "completed" && usage.failed { + "error" + } else { + status + }; + // Seconds keep multi-minute turns within the standard histogram buckets. + // Round up so integer-second bucket boundaries classify fractional durations correctly. + let duration_seconds = self.started_at.elapsed().as_secs_f64().ceil() as i64; + for plugin_id in usage.plugins { + let plugin_id = sanitize_metric_tag_value(&plugin_id); + metrics.histogram( + "codex.skill.turn.duration_seconds", + duration_seconds, + &[ + ("plugin_id", plugin_id.as_str()), + ("model_slug", self.model_slug.as_str()), + ("reasoning_effort", self.reasoning_effort.as_str()), + ("status", status), + ], + ); + } + } +} + +#[derive(Default)] +pub(crate) struct ActiveSkillTurnMetrics(pub(crate) Mutex>); + +pub(crate) struct SkillTelemetry; + +impl TurnLifecycleContributor for SkillTelemetry { + fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + let reasoning_effort = input + .collaboration_mode + .reasoning_effort() + .or_else(|| { + input + .thread_store + .get::() + .and_then(|model| model.default_reasoning_level.clone()) + }) + .map(|effort| effort.to_string()) + .unwrap_or_else(|| "default".to_string()); + input.turn_store.insert(SkillTurnMetrics { + turn_id: input.turn_id.to_string(), + model_slug: sanitize_metric_tag_value(input.collaboration_mode.model()), + reasoning_effort, + started_at: Instant::now(), + usage: Mutex::new(Some(TurnUsage::default())), + }); + if let Some(turn) = input.turn_store.get::() { + *input + .thread_store + .get_or_init(ActiveSkillTurnMetrics::default) + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::downgrade(&turn); + } + }) + } + + fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if let Some(turn) = input.turn_store.get::() { + turn.finish(input.session_store, "completed"); + } + }) + } + + fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if let Some(turn) = input.turn_store.get::() { + turn.finish(input.session_store, "aborted"); + } + }) + } + + fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> { + Box::pin(async move { + if let Some(turn) = input.turn_store.get::() + && let Some(usage) = turn + .usage + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_mut() + { + usage.failed = true; + } + }) + } +} diff --git a/codex-rs/ext/skills/src/tools/mod.rs b/codex-rs/ext/skills/src/tools/mod.rs index 4869c5f497..54d663e2a5 100644 --- a/codex-rs/ext/skills/src/tools/mod.rs +++ b/codex-rs/ext/skills/src/tools/mod.rs @@ -11,6 +11,7 @@ use codex_analytics::SkillInvocationLocation; use codex_analytics::build_track_events_context; use codex_exec_server::FileSystemSandboxContext; use codex_extension_api::ExtensionData; +use codex_extension_api::ExtensionMetrics; use codex_extension_api::FunctionCallError; use codex_extension_api::JsonToolOutput; use codex_extension_api::ResponsesApiTool; @@ -24,6 +25,7 @@ use codex_extension_api::ToolSpec; use codex_extension_api::parse_tool_input_schema; use codex_mcp::CODEX_APPS_MCP_SERVER_NAME; use codex_mcp::McpResourceClient; +use codex_otel::sanitize_metric_tag_value; use codex_tools::ResponsesApiNamespace; use codex_tools::ResponsesApiNamespaceTool; use codex_tools::default_namespace_description; @@ -43,6 +45,7 @@ use crate::shadow_selection_experiment::ShadowSelectionExperiment; use crate::sources::SkillProviders; use crate::state::SkillsSessionState; use crate::state::SkillsThreadState; +use crate::telemetry::ActiveSkillTurnMetrics; mod list; mod read; @@ -96,6 +99,8 @@ pub(crate) fn skill_tools( #[derive(Clone)] pub(crate) struct SkillAnalytics { client: AnalyticsEventsClient, + metrics: Option>, + active_turn: Arc, thread_id: String, product_client_id: String, } @@ -110,6 +115,10 @@ impl SkillAnalytics { Some(Self { client: client.as_ref().clone(), + metrics: session_store + .get::() + .and_then(|state| state.extension_metrics.clone()), + active_turn: thread_store.get_or_init(ActiveSkillTurnMetrics::default), thread_id: thread_store.level_id().to_string(), product_client_id: originator.0.clone(), }) @@ -122,6 +131,42 @@ impl SkillAnalytics { turn_id: String, invocation_type: InvocationType, ) { + let turn_metrics = self + .active_turn + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .upgrade() + .filter(|turn| turn.turn_id == turn_id); + if let Some(turn_metrics) = &turn_metrics { + turn_metrics.record_plugin(skill.plugin_id.as_deref()); + } + if let Some(metrics) = &self.metrics { + let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str()); + let plugin_id_tag = + sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed")); + let model_slug_tag = sanitize_metric_tag_value(model.as_str()); + let reasoning_effort = turn_metrics + .as_ref() + .map(|turn| turn.reasoning_effort.as_str()) + .unwrap_or("unknown"); + let invoke_type = match invocation_type { + InvocationType::Explicit => "explicit", + InvocationType::Implicit => "implicit", + }; + metrics.counter( + "codex.skill.injected", + /*inc*/ 1, + &[ + ("status", "ok"), + ("skill", skill_name_tag.as_str()), + ("invoke_type", invoke_type), + ("plugin_id", plugin_id_tag.as_str()), + ("model_slug", model_slug_tag.as_str()), + ("reasoning_effort", reasoning_effort), + ], + ); + } self.client.track_skill_invocations( build_track_events_context( model,