openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-20 17:38:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add plugin-attributed skill telemetry (#40724)

Browse Source 
## What changed

- Add `plugin_id`, `model_slug`, and `reasoning_effort` dimensions to
  `codex.skill.injected` metrics for explicit and implicit skill invocations.
- Propagate plugin IDs from orchestrator skill metadata so resource-backed skill
  invocations can emit the same attribution.
- Record `codex.skill.turn.duration_seconds` once per plugin used in a turn,
  tagged with the model, reasoning effort, and completed, aborted, or error
  status.

GitOrigin-RevId: d249a9c4e85682b1106ac49d9e598535c6cbb06d
This commit is contained in:
jwang-openai
2026-08-25 22:29:14 +00:00
committed by copyberry
parent 62bfa41a83
commit c51e7b3736
6 changed files with 235 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/skills.rs
+23
View File
@@ -13,6 +13,7 @@ use codex_protocol::protocol::SkillScope;
use codex_skills::SkillMetadata;
use codex_skills_extension::HostSkillsLoadInput;
use codex_skills_extension::detect_implicit_skill_invocation;
use codex_skills_extension::record_plugin_turn_usage;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use codex_utils_plugins::PluginSkillRoot;
@@ -44,9 +45,17 @@ pub(crate) fn emit_explicit_skill_invocations(
.iter()
.map(|skill| &skill.path_to_skills_md)
.collect::<HashSet<_>>();
let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());
let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();
for skill in mentioned_skills {
let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());
let plugin_id_tag =
sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));
let status = if injected_skill_paths.contains(&skill.path_to_skills_md) {
record_plugin_turn_usage(
turn_context.extension_data.as_ref(),
skill.plugin_id.as_deref(),
);
"ok"
} else {
"error"
@@ -58,6 +67,9 @@ pub(crate) fn emit_explicit_skill_invocations(
("status", status),
("skill", skill_name_tag.as_str()),
("invoke_type", "explicit"),
("plugin_id", plugin_id_tag.as_str()),
("model_slug", model_slug_tag.as_str()),
("reasoning_effort", reasoning_effort.as_str()),
],
);
}
@@ -123,6 +135,14 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
return;
}
let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str());
let plugin_id_tag =
sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed"));
let model_slug_tag = sanitize_metric_tag_value(turn_context.model_info.slug.as_str());
let reasoning_effort = turn_context.effective_reasoning_effort_for_tracing();
record_plugin_turn_usage(
turn_context.extension_data.as_ref(),
invocation.plugin_id.as_deref(),
);
for contributor in sess.services.extensions.skill_invocation_contributors() {
contributor
@@ -144,6 +164,9 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
("status", "ok"),
("skill", skill_name_tag.as_str()),
("invoke_type", "implicit"),
("plugin_id", plugin_id_tag.as_str()),
("model_slug", model_slug_tag.as_str()),
("reasoning_effort", reasoning_effort.as_str()),
],
);
sess.services
codex-rs/ext/skills/src/extension.rs
+2
View File
@@ -67,6 +67,7 @@ use crate::state::HostSkillsStepState;
use crate::state::SkillsSessionState;
use crate::state::SkillsThreadState;
use crate::state::SkillsTurnState;
use crate::telemetry::SkillTelemetry;
use crate::tools::SkillAnalytics;
use crate::tools::SkillToolAuthority;
use crate::tools::skill_tools;
@@ -670,6 +671,7 @@ pub fn install_with_providers_and_metrics<C>(
shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)),
});
registry.thread_lifecycle_contributor(extension.clone());
registry.turn_lifecycle_contributor(Arc::new(SkillTelemetry));
registry.config_contributor(extension.clone());
registry.prompt_contributor(extension.clone());
registry.turn_input_contributor(extension.clone());
codex-rs/ext/skills/src/lib.rs
+2
View File
@@ -20,6 +20,7 @@ mod selection;
mod shadow_selection_experiment;
mod sources;
mod state;
mod telemetry;
mod tools;
mod warnings;
mod world_state;
@@ -43,6 +44,7 @@ pub use provider::OrchestratorSkillProvider;
pub use provider::SkillProvider;
pub use sources::SkillProviderSource;
pub use sources::SkillProviders;
pub use telemetry::record_plugin_turn_usage;
/// Recognizes persisted explicit skill prompts without exposing their fragment implementation.
pub fn is_skill_prompt_fragment(text: &str) -> bool {
codex-rs/ext/skills/src/provider/orchestrator.rs
+4
View File
@@ -276,6 +276,10 @@ fn catalog_entry_from_resource(resource: &Resource) -> Option<SkillCatalogEntry>
)
.with_display_path(uri)
.with_alias_root(format!("skill://{namespace}"));
entry.plugin_id = meta
.get("plugin_id")
.and_then(serde_json::Value::as_str)
.map(str::to_owned);
entry.canonical_skill_id = meta
.get("skill_id")
.and_then(serde_json::Value::as_str)
codex-rs/ext/skills/src/telemetry.rs
+159
View File
@@ -0,0 +1,159 @@
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::Weak;
use std::time::Instant;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionFuture;
use codex_extension_api::TurnAbortInput;
use codex_extension_api::TurnErrorInput;
use codex_extension_api::TurnLifecycleContributor;
use codex_extension_api::TurnStartInput;
use codex_extension_api::TurnStopInput;
use codex_otel::sanitize_metric_tag_value;
use codex_protocol::openai_models::ModelInfo;
use crate::state::SkillsSessionState;
/// Attributes a successful host skill invocation to its turn's plugin latency metrics.
pub fn record_plugin_turn_usage(turn_store: &ExtensionData, plugin_id: Option<&str>) {
if let Some(turn) = turn_store.get::<SkillTurnMetrics>() {
turn.record_plugin(plugin_id);
}
}
pub(crate) struct SkillTurnMetrics {
pub(crate) turn_id: String,
model_slug: String,
pub(crate) reasoning_effort: String,
started_at: Instant,
usage: Mutex<Option<TurnUsage>>,
}
#[derive(Default)]
struct TurnUsage {
plugins: HashSet<String>,
failed: bool,
}
impl SkillTurnMetrics {
pub(crate) fn record_plugin(&self, plugin_id: Option<&str>) {
if let Some(usage) = self
.usage
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_mut()
{
usage
.plugins
.insert(plugin_id.unwrap_or("unattributed").to_string());
}
}
fn finish(&self, session_store: &ExtensionData, status: &str) {
let Some(usage) = self
.usage
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
else {
return;
};
let Some(metrics) = session_store
.get::<SkillsSessionState>()
.and_then(|state| state.extension_metrics.clone())
else {
return;
};
let status = if status == "completed" && usage.failed {
"error"
} else {
status
};
// Seconds keep multi-minute turns within the standard histogram buckets.
// Round up so integer-second bucket boundaries classify fractional durations correctly.
let duration_seconds = self.started_at.elapsed().as_secs_f64().ceil() as i64;
for plugin_id in usage.plugins {
let plugin_id = sanitize_metric_tag_value(&plugin_id);
metrics.histogram(
"codex.skill.turn.duration_seconds",
duration_seconds,
&[
("plugin_id", plugin_id.as_str()),
("model_slug", self.model_slug.as_str()),
("reasoning_effort", self.reasoning_effort.as_str()),
("status", status),
],
);
}
}
}
#[derive(Default)]
pub(crate) struct ActiveSkillTurnMetrics(pub(crate) Mutex<Weak<SkillTurnMetrics>>);
pub(crate) struct SkillTelemetry;
impl TurnLifecycleContributor for SkillTelemetry {
fn on_turn_start<'a>(&'a self, input: TurnStartInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
let reasoning_effort = input
.collaboration_mode
.reasoning_effort()
.or_else(|| {
input
.thread_store
.get::<ModelInfo>()
.and_then(|model| model.default_reasoning_level.clone())
})
.map(|effort| effort.to_string())
.unwrap_or_else(|| "default".to_string());
input.turn_store.insert(SkillTurnMetrics {
turn_id: input.turn_id.to_string(),
model_slug: sanitize_metric_tag_value(input.collaboration_mode.model()),
reasoning_effort,
started_at: Instant::now(),
usage: Mutex::new(Some(TurnUsage::default())),
});
if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
*input
.thread_store
.get_or_init(ActiveSkillTurnMetrics::default)
.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Arc::downgrade(&turn);
}
})
}
fn on_turn_stop<'a>(&'a self, input: TurnStopInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
turn.finish(input.session_store, "completed");
}
})
}
fn on_turn_abort<'a>(&'a self, input: TurnAbortInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>() {
turn.finish(input.session_store, "aborted");
}
})
}
fn on_turn_error<'a>(&'a self, input: TurnErrorInput<'a>) -> ExtensionFuture<'a, ()> {
Box::pin(async move {
if let Some(turn) = input.turn_store.get::<SkillTurnMetrics>()
&& let Some(usage) = turn
.usage
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_mut()
{
usage.failed = true;
}
})
}
}
codex-rs/ext/skills/src/tools/mod.rs
+45
View File
@@ -11,6 +11,7 @@ use codex_analytics::SkillInvocationLocation;
use codex_analytics::build_track_events_context;
use codex_exec_server::FileSystemSandboxContext;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionMetrics;
use codex_extension_api::FunctionCallError;
use codex_extension_api::JsonToolOutput;
use codex_extension_api::ResponsesApiTool;
@@ -24,6 +25,7 @@ use codex_extension_api::ToolSpec;
use codex_extension_api::parse_tool_input_schema;
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
use codex_mcp::McpResourceClient;
use codex_otel::sanitize_metric_tag_value;
use codex_tools::ResponsesApiNamespace;
use codex_tools::ResponsesApiNamespaceTool;
use codex_tools::default_namespace_description;
@@ -43,6 +45,7 @@ use crate::shadow_selection_experiment::ShadowSelectionExperiment;
use crate::sources::SkillProviders;
use crate::state::SkillsSessionState;
use crate::state::SkillsThreadState;
use crate::telemetry::ActiveSkillTurnMetrics;
mod list;
mod read;
@@ -96,6 +99,8 @@ pub(crate) fn skill_tools(
#[derive(Clone)]
pub(crate) struct SkillAnalytics {
client: AnalyticsEventsClient,
metrics: Option<Arc<dyn ExtensionMetrics>>,
active_turn: Arc<ActiveSkillTurnMetrics>,
thread_id: String,
product_client_id: String,
}
@@ -110,6 +115,10 @@ impl SkillAnalytics {
Some(Self {
client: client.as_ref().clone(),
metrics: session_store
.get::<SkillsSessionState>()
.and_then(|state| state.extension_metrics.clone()),
active_turn: thread_store.get_or_init(ActiveSkillTurnMetrics::default),
thread_id: thread_store.level_id().to_string(),
product_client_id: originator.0.clone(),
})
@@ -122,6 +131,42 @@ impl SkillAnalytics {
turn_id: String,
invocation_type: InvocationType,
) {
let turn_metrics = self
.active_turn
.0
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.upgrade()
.filter(|turn| turn.turn_id == turn_id);
if let Some(turn_metrics) = &turn_metrics {
turn_metrics.record_plugin(skill.plugin_id.as_deref());
}
if let Some(metrics) = &self.metrics {
let skill_name_tag = sanitize_metric_tag_value(skill.name.as_str());
let plugin_id_tag =
sanitize_metric_tag_value(skill.plugin_id.as_deref().unwrap_or("unattributed"));
let model_slug_tag = sanitize_metric_tag_value(model.as_str());
let reasoning_effort = turn_metrics
.as_ref()
.map(|turn| turn.reasoning_effort.as_str())
.unwrap_or("unknown");
let invoke_type = match invocation_type {
InvocationType::Explicit => "explicit",
InvocationType::Implicit => "implicit",
};
metrics.counter(
"codex.skill.injected",
/*inc*/ 1,
&[
("status", "ok"),
("skill", skill_name_tag.as_str()),
("invoke_type", invoke_type),
("plugin_id", plugin_id_tag.as_str()),
("model_slug", model_slug_tag.as_str()),
("reasoning_effort", reasoning_effort),
],
);
}
self.client.track_skill_invocations(
build_track_events_context(
model,