mirror of
https://github.com/openai/codex.git
synced 2026-10-18 17:18:52 +00:00
Export skill invocation events through OpenTelemetry (#49689)
## What changed Emit `codex.skill_invocation` logs for explicit skill injection and detected implicit use, including skill-tool reads. Include skill name, invocation type, conversation, turn, model, and client metadata, plus scope, plugin ID, and ChatGPT user and account IDs when available. Exclude skill contents, resource paths, email addresses, prompts, and tool payloads. Use the existing OTEL log exporter without an additional opt-in; `otel.exporter = "none"` continues to export no logs. Document the event fields, exporter configuration, and detection semantics in `codex-rs/otel/README.md`. GitOrigin-RevId: 0f9c492c3f224455f3de3660be89692d776c1951
This commit is contained in:
@@ -1303,6 +1303,7 @@ impl Session {
|
||||
session_configuration.session_source.clone(),
|
||||
)
|
||||
.with_auth_env(auth_env_telemetry.to_otel_metadata())
|
||||
.with_user_id(telemetry_auth.and_then(CodexAuth::get_chatgpt_user_id))
|
||||
.with_tool_result_log_config(config.otel.tool_result);
|
||||
if let Some(metrics) = thread_extension_data.get::<codex_otel::MetricsClient>() {
|
||||
session_telemetry = session_telemetry.with_metrics(metrics.as_ref().clone());
|
||||
@@ -1665,6 +1666,7 @@ impl Session {
|
||||
let session_extension_data =
|
||||
codex_extension_api::ExtensionData::new(session_id.to_string());
|
||||
session_extension_data.insert(analytics_events_client.clone());
|
||||
session_extension_data.insert(session_telemetry.clone());
|
||||
let mcp_resource_client = Arc::new(McpResourceClient::new(Arc::clone(&mcp_runtime)));
|
||||
let extension_metrics =
|
||||
extension_metrics::from_session_telemetry(session_telemetry.clone());
|
||||
|
||||
@@ -8,6 +8,8 @@ use codex_analytics::TrackEventsContext;
|
||||
use codex_analytics::build_track_events_context;
|
||||
use codex_extension_api::SkillInvocationInput;
|
||||
use codex_extension_api::SkillInvocationKind;
|
||||
use codex_otel::SkillInvocationEvent;
|
||||
use codex_otel::SkillInvocationType;
|
||||
use codex_otel::sanitize_metric_tag_value;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
use codex_skills::SkillMetadata;
|
||||
@@ -86,6 +88,15 @@ pub(crate) async fn emit_explicit_skill_invocations(
|
||||
{
|
||||
continue;
|
||||
}
|
||||
turn_context
|
||||
.session_telemetry
|
||||
.skill_invocation(SkillInvocationEvent {
|
||||
turn_id: &turn_context.sub_id,
|
||||
skill_name: &skill.name,
|
||||
scope: Some(skill.scope),
|
||||
plugin_id: skill.plugin_id.as_deref(),
|
||||
invocation_type: SkillInvocationType::Explicit,
|
||||
});
|
||||
for contributor in sess.services.extensions.skill_invocation_contributors() {
|
||||
contributor
|
||||
.on_skill_invocation(SkillInvocationInput {
|
||||
@@ -160,6 +171,18 @@ pub(crate) async fn maybe_emit_implicit_skill_invocation(
|
||||
if !inserted {
|
||||
return;
|
||||
}
|
||||
turn_context
|
||||
.session_telemetry
|
||||
.skill_invocation(SkillInvocationEvent {
|
||||
turn_id: &turn_context.sub_id,
|
||||
skill_name: &skill_name,
|
||||
scope: match &invocation.location {
|
||||
SkillInvocationLocation::Host { scope, .. } => Some(*scope),
|
||||
SkillInvocationLocation::Resource { scope, .. } => *scope,
|
||||
},
|
||||
plugin_id: invocation.plugin_id.as_deref(),
|
||||
invocation_type: SkillInvocationType::Implicit,
|
||||
});
|
||||
let skill_name_tag = sanitize_metric_tag_value(skill_name.as_str());
|
||||
let plugin_id_tag =
|
||||
sanitize_metric_tag_value(invocation.plugin_id.as_deref().unwrap_or("unattributed"));
|
||||
|
||||
@@ -25,6 +25,9 @@ use codex_extension_api::ToolSpec;
|
||||
use codex_extension_api::parse_tool_input_schema;
|
||||
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
||||
use codex_mcp::McpResourceClient;
|
||||
use codex_otel::SessionTelemetry;
|
||||
use codex_otel::SkillInvocationEvent;
|
||||
use codex_otel::SkillInvocationType;
|
||||
use codex_otel::sanitize_metric_tag_value;
|
||||
use codex_tools::ResponsesApiNamespace;
|
||||
use codex_tools::ResponsesApiNamespaceTool;
|
||||
@@ -99,6 +102,7 @@ pub(crate) fn skill_tools(
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SkillAnalytics {
|
||||
client: AnalyticsEventsClient,
|
||||
telemetry: Option<Arc<SessionTelemetry>>,
|
||||
metrics: Option<Arc<dyn ExtensionMetrics>>,
|
||||
turn_metrics: Option<Arc<SkillTurnMetrics>>,
|
||||
thread_id: String,
|
||||
@@ -115,6 +119,7 @@ impl SkillAnalytics {
|
||||
|
||||
Some(Self {
|
||||
client: client.as_ref().clone(),
|
||||
telemetry: session_store.get::<SessionTelemetry>(),
|
||||
metrics: session_store
|
||||
.get::<SkillsSessionState>()
|
||||
.and_then(|state| state.extension_metrics.clone()),
|
||||
@@ -137,6 +142,22 @@ impl SkillAnalytics {
|
||||
turn_id: String,
|
||||
invocation_type: InvocationType,
|
||||
) {
|
||||
if let Some(telemetry) = &self.telemetry {
|
||||
telemetry
|
||||
.as_ref()
|
||||
.clone()
|
||||
.with_model(&model, &model)
|
||||
.skill_invocation(SkillInvocationEvent {
|
||||
turn_id: &turn_id,
|
||||
skill_name: &skill.name,
|
||||
scope: skill.analytics_scope,
|
||||
plugin_id: skill.plugin_id.as_deref(),
|
||||
invocation_type: match invocation_type {
|
||||
InvocationType::Explicit => SkillInvocationType::Explicit,
|
||||
InvocationType::Implicit => SkillInvocationType::Implicit,
|
||||
},
|
||||
});
|
||||
}
|
||||
let turn_metrics = self
|
||||
.turn_metrics
|
||||
.as_ref()
|
||||
|
||||
@@ -96,6 +96,32 @@ let manager = SessionTelemetry::new(
|
||||
manager.user_prompt(&prompt_items);
|
||||
```
|
||||
|
||||
### Skill invocation logs
|
||||
|
||||
`codex.skill_invocation` records explicit skill injection and detected implicit use,
|
||||
including reads through the skills tools. It contains `skill.name`,
|
||||
`skill.invocation_type` (`explicit` or `implicit`), `conversation.id`, `turn.id`,
|
||||
model and client metadata, and available `skill.scope`, `skill.plugin_id`,
|
||||
`user.id`, and `user.account_id`. The user ID identifies the authenticated ChatGPT
|
||||
user; the account ID identifies the workspace. Missing identity fields are omitted,
|
||||
including the user ID for API-key authentication.
|
||||
|
||||
The event excludes skill contents, descriptions, resource paths, email addresses,
|
||||
prompts, tool arguments, and tool output. It reports detected use, not successful
|
||||
task completion. Shell-based implicit detection is best effort and is deduplicated
|
||||
per skill within a turn; explicit injection and later implicit access are distinct
|
||||
events. Skill-tool reads retain their existing invocation behavior.
|
||||
|
||||
Skill events use the existing OTEL log exporter and are included automatically
|
||||
when `otel.exporter` points to an OTLP HTTP or gRPC destination. The default
|
||||
`otel.exporter = "none"` exports no logs. No additional configuration or skill-event
|
||||
opt-in is required; the destination also receives the other existing log events.
|
||||
|
||||
```toml
|
||||
[otel]
|
||||
exporter = { otlp-http = { endpoint = "https://collector.example.com/v1/logs", protocol = "json" } }
|
||||
```
|
||||
|
||||
### Agent response logs
|
||||
|
||||
Set `otel.log_agent_responses = true` with an OTLP HTTP or gRPC log exporter to
|
||||
|
||||
@@ -100,6 +100,7 @@ pub struct SessionTelemetryMetadata {
|
||||
pub(crate) auth_mode: Option<String>,
|
||||
pub(crate) auth_env: AuthEnvTelemetryMetadata,
|
||||
pub(crate) account_id: Option<String>,
|
||||
pub(crate) user_id: Option<String>,
|
||||
pub(crate) account_email: Option<String>,
|
||||
pub(crate) originator: String,
|
||||
pub(crate) product_sku: Option<&'static str>,
|
||||
@@ -587,6 +588,7 @@ impl SessionTelemetry {
|
||||
auth_mode: auth_mode.map(|m| m.to_string()),
|
||||
auth_env: AuthEnvTelemetryMetadata::default(),
|
||||
account_id,
|
||||
user_id: None,
|
||||
account_email,
|
||||
originator: sanitize_metric_tag_value(originator.as_str()),
|
||||
product_sku: None,
|
||||
|
||||
@@ -9,6 +9,7 @@ mod agent_response;
|
||||
mod guardian_assessment;
|
||||
mod network_policy;
|
||||
mod otlp;
|
||||
mod skill_invocation;
|
||||
mod targets;
|
||||
mod tool_result;
|
||||
|
||||
@@ -33,6 +34,8 @@ pub use crate::metrics::runtime_metrics::RuntimeMetricsSummary;
|
||||
pub use crate::metrics::timer::Timer;
|
||||
pub use crate::metrics::*;
|
||||
pub use crate::provider::OtelProvider;
|
||||
pub use crate::skill_invocation::SkillInvocationEvent;
|
||||
pub use crate::skill_invocation::SkillInvocationType;
|
||||
pub use crate::trace_context::context_from_w3c_trace_context;
|
||||
pub use crate::trace_context::current_span_trace_id;
|
||||
pub use crate::trace_context::current_span_w3c_trace_context;
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
//! Exports skill-use metadata without skill contents, filesystem paths, or tool payloads.
|
||||
//! Detection belongs to the caller; an invocation does not imply successful task completion.
|
||||
|
||||
use crate::SessionTelemetry;
|
||||
use crate::events::shared::timestamp;
|
||||
use crate::targets::OTEL_LOG_ONLY_TARGET;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
|
||||
/// How Codex detected the skill's use.
|
||||
pub enum SkillInvocationType {
|
||||
Explicit,
|
||||
Implicit,
|
||||
}
|
||||
|
||||
/// Metadata for a detected skill invocation. Never include skill contents or resource paths.
|
||||
pub struct SkillInvocationEvent<'a> {
|
||||
pub turn_id: &'a str,
|
||||
pub skill_name: &'a str,
|
||||
pub scope: Option<SkillScope>,
|
||||
pub plugin_id: Option<&'a str>,
|
||||
pub invocation_type: SkillInvocationType,
|
||||
}
|
||||
|
||||
impl SessionTelemetry {
|
||||
pub fn with_user_id(mut self, user_id: Option<String>) -> Self {
|
||||
self.metadata.user_id = user_id;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn skill_invocation(&self, event: SkillInvocationEvent<'_>) {
|
||||
let invocation_type = match event.invocation_type {
|
||||
SkillInvocationType::Explicit => "explicit",
|
||||
SkillInvocationType::Implicit => "implicit",
|
||||
};
|
||||
let scope = event.scope.map(|scope| match scope {
|
||||
SkillScope::User => "user",
|
||||
SkillScope::Repo => "repo",
|
||||
SkillScope::System => "system",
|
||||
SkillScope::Admin => "admin",
|
||||
});
|
||||
tracing::event!(
|
||||
name: "codex.skill_invocation",
|
||||
target: OTEL_LOG_ONLY_TARGET,
|
||||
tracing::Level::INFO,
|
||||
event.name = "codex.skill_invocation",
|
||||
event.timestamp = %timestamp(),
|
||||
conversation.id = %self.metadata.conversation_id,
|
||||
turn.id = event.turn_id,
|
||||
user.id = self.metadata.user_id.as_deref(),
|
||||
user.account_id = self.metadata.account_id.as_deref(),
|
||||
skill.name = event.skill_name,
|
||||
skill.scope = scope,
|
||||
skill.plugin_id = event.plugin_id,
|
||||
skill.invocation_type = invocation_type,
|
||||
model = %self.metadata.model,
|
||||
app.version = self.metadata.app_version,
|
||||
originator = %self.metadata.originator,
|
||||
auth_mode = self.metadata.auth_mode.as_deref(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user