From 1661767afbbc5b11994e297ebdade07be9c08d50 Mon Sep 17 00:00:00 2001 From: Ahmed Ibrahim Date: Wed, 25 Mar 2026 00:05:47 -0700 Subject: [PATCH] fix --- codex-rs/analytics/BUILD.bazel | 6 + codex-rs/analytics/Cargo.toml | 31 + codex-rs/analytics/src/analytics_client.rs | 797 ++++++++++++++++++ .../analytics/src/analytics_client_tests.rs | 290 +++++++ codex-rs/analytics/src/lib.rs | 8 + codex-rs/config/src/project_root_markers.rs | 50 ++ .../core-skills/src/skills/mention_counts.rs | 24 + codex-rs/core/src/analytics_client.rs.bak | 0 codex-rs/core/src/plugins/mentions.rs | 121 +++ codex-rs/core/src/plugins/mentions_tests.rs | 155 ++++ codex-rs/instructions/BUILD.bazel | 16 + codex-rs/instructions/Cargo.toml | 20 + codex-rs/instructions/src/fragment.rs | 61 ++ codex-rs/instructions/src/lib.rs | 15 + .../instructions/src/user_instructions.rs | 56 ++ .../src/user_instructions_tests.rs | 72 ++ codex-rs/plugin/src/mention_syntax.rs | 7 + codex-rs/plugin/src/plugin_namespace.rs | 70 ++ 18 files changed, 1799 insertions(+) create mode 100644 codex-rs/analytics/BUILD.bazel create mode 100644 codex-rs/analytics/Cargo.toml create mode 100644 codex-rs/analytics/src/analytics_client.rs create mode 100644 codex-rs/analytics/src/analytics_client_tests.rs create mode 100644 codex-rs/analytics/src/lib.rs create mode 100644 codex-rs/config/src/project_root_markers.rs create mode 100644 codex-rs/core-skills/src/skills/mention_counts.rs create mode 100644 codex-rs/core/src/analytics_client.rs.bak create mode 100644 codex-rs/core/src/plugins/mentions.rs create mode 100644 codex-rs/core/src/plugins/mentions_tests.rs create mode 100644 codex-rs/instructions/BUILD.bazel create mode 100644 codex-rs/instructions/Cargo.toml create mode 100644 codex-rs/instructions/src/fragment.rs create mode 100644 codex-rs/instructions/src/lib.rs create mode 100644 codex-rs/instructions/src/user_instructions.rs create mode 100644 codex-rs/instructions/src/user_instructions_tests.rs create mode 100644 codex-rs/plugin/src/mention_syntax.rs create mode 100644 codex-rs/plugin/src/plugin_namespace.rs diff --git a/codex-rs/analytics/BUILD.bazel b/codex-rs/analytics/BUILD.bazel new file mode 100644 index 0000000000..aec07c8746 --- /dev/null +++ b/codex-rs/analytics/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "analytics", + crate_name = "codex_analytics", +) diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml new file mode 100644 index 0000000000..22ae170854 --- /dev/null +++ b/codex-rs/analytics/Cargo.toml @@ -0,0 +1,31 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-analytics" +version.workspace = true + +[lib] +doctest = false +name = "codex_analytics" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-git-utils = { workspace = true } +codex-login = { workspace = true } +codex-plugin = { workspace = true } +codex-protocol = { workspace = true } +reqwest = { workspace = true, features = ["json", "stream"] } +serde = { workspace = true, features = ["derive"] } +sha1 = { workspace = true } +tokio = { workspace = true, features = [ + "macros", + "rt-multi-thread", +] } +tracing = { workspace = true, features = ["log"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } diff --git a/codex-rs/analytics/src/analytics_client.rs b/codex-rs/analytics/src/analytics_client.rs new file mode 100644 index 0000000000..ac8d10b6c1 --- /dev/null +++ b/codex-rs/analytics/src/analytics_client.rs @@ -0,0 +1,797 @@ +use codex_git_utils::collect_git_info; +use codex_git_utils::get_git_repo_root; +use codex_login::AuthManager; +use codex_login::default_client::create_client; +use codex_login::default_client::originator; +use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::protocol::SkillScope; +use serde::Serialize; +use sha1::Digest; +use sha1::Sha1; +use std::collections::HashSet; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Duration; +use tokio::sync::mpsc; + +#[derive(Clone)] +pub struct TrackEventsContext { + pub model_slug: String, + pub thread_id: String, + pub turn_id: String, +} + +pub fn build_track_events_context( + model_slug: String, + thread_id: String, + turn_id: String, +) -> TrackEventsContext { + TrackEventsContext { + model_slug, + thread_id, + turn_id, + } +} + +#[derive(Clone, Debug)] +pub struct SkillInvocation { + pub skill_name: String, + pub skill_scope: SkillScope, + pub skill_path: PathBuf, + pub invocation_type: InvocationType, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum InvocationType { + Explicit, + Implicit, +} + +pub struct AppInvocation { + pub connector_id: Option, + pub app_name: Option, + pub invocation_type: Option, +} + +#[derive(Clone)] +pub(crate) struct AnalyticsEventsQueue { + sender: mpsc::Sender, + app_used_emitted_keys: Arc>>, + plugin_used_emitted_keys: Arc>>, +} + +#[derive(Clone)] +pub struct AnalyticsEventsClient { + queue: AnalyticsEventsQueue, + analytics_enabled: Option, +} + +impl AnalyticsEventsQueue { + pub(crate) fn new(auth_manager: Arc, base_url: String) -> Self { + let (sender, mut receiver) = mpsc::channel(ANALYTICS_EVENTS_QUEUE_SIZE); + tokio::spawn(async move { + while let Some(job) = receiver.recv().await { + match job { + TrackEventsJob::SkillInvocations(job) => { + send_track_skill_invocations(&auth_manager, &base_url, job).await; + } + TrackEventsJob::AppMentioned(job) => { + send_track_app_mentioned(&auth_manager, &base_url, job).await; + } + TrackEventsJob::AppUsed(job) => { + send_track_app_used(&auth_manager, &base_url, job).await; + } + TrackEventsJob::PluginUsed(job) => { + send_track_plugin_used(&auth_manager, &base_url, job).await; + } + TrackEventsJob::PluginInstalled(job) => { + send_track_plugin_installed(&auth_manager, &base_url, job).await; + } + TrackEventsJob::PluginUninstalled(job) => { + send_track_plugin_uninstalled(&auth_manager, &base_url, job).await; + } + TrackEventsJob::PluginEnabled(job) => { + send_track_plugin_enabled(&auth_manager, &base_url, job).await; + } + TrackEventsJob::PluginDisabled(job) => { + send_track_plugin_disabled(&auth_manager, &base_url, job).await; + } + } + } + }); + Self { + sender, + app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + plugin_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + } + } + + fn try_send(&self, job: TrackEventsJob) { + if self.sender.try_send(job).is_err() { + //TODO: add a metric for this + tracing::warn!("dropping analytics events: queue is full"); + } + } + + fn should_enqueue_app_used(&self, tracking: &TrackEventsContext, app: &AppInvocation) -> bool { + let Some(connector_id) = app.connector_id.as_ref() else { + return true; + }; + let mut emitted = self + .app_used_emitted_keys + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if emitted.len() >= ANALYTICS_EVENT_DEDUPE_MAX_KEYS { + emitted.clear(); + } + emitted.insert((tracking.turn_id.clone(), connector_id.clone())) + } + + fn should_enqueue_plugin_used( + &self, + tracking: &TrackEventsContext, + plugin: &PluginTelemetryMetadata, + ) -> bool { + let mut emitted = self + .plugin_used_emitted_keys + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if emitted.len() >= ANALYTICS_EVENT_DEDUPE_MAX_KEYS { + emitted.clear(); + } + emitted.insert((tracking.turn_id.clone(), plugin.plugin_id.as_key())) + } +} + +impl AnalyticsEventsClient { + pub fn new( + auth_manager: Arc, + base_url: String, + analytics_enabled: Option, + ) -> Self { + Self { + queue: AnalyticsEventsQueue::new(Arc::clone(&auth_manager), base_url), + analytics_enabled, + } + } + + pub fn track_skill_invocations( + &self, + tracking: TrackEventsContext, + invocations: Vec, + ) { + track_skill_invocations( + &self.queue, + self.analytics_enabled, + Some(tracking), + invocations, + ); + } + + pub fn track_app_mentioned(&self, tracking: TrackEventsContext, mentions: Vec) { + track_app_mentioned( + &self.queue, + self.analytics_enabled, + Some(tracking), + mentions, + ); + } + + pub fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) { + track_app_used(&self.queue, self.analytics_enabled, Some(tracking), app); + } + + pub fn track_plugin_used(&self, tracking: TrackEventsContext, plugin: PluginTelemetryMetadata) { + track_plugin_used(&self.queue, self.analytics_enabled, Some(tracking), plugin); + } + + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { + track_plugin_management( + &self.queue, + self.analytics_enabled, + PluginManagementEventType::Installed, + plugin, + ); + } + + pub fn track_plugin_uninstalled(&self, plugin: PluginTelemetryMetadata) { + track_plugin_management( + &self.queue, + self.analytics_enabled, + PluginManagementEventType::Uninstalled, + plugin, + ); + } + + pub fn track_plugin_enabled(&self, plugin: PluginTelemetryMetadata) { + track_plugin_management( + &self.queue, + self.analytics_enabled, + PluginManagementEventType::Enabled, + plugin, + ); + } + + pub fn track_plugin_disabled(&self, plugin: PluginTelemetryMetadata) { + track_plugin_management( + &self.queue, + self.analytics_enabled, + PluginManagementEventType::Disabled, + plugin, + ); + } +} + +enum TrackEventsJob { + SkillInvocations(TrackSkillInvocationsJob), + AppMentioned(TrackAppMentionedJob), + AppUsed(TrackAppUsedJob), + PluginUsed(TrackPluginUsedJob), + PluginInstalled(TrackPluginManagementJob), + PluginUninstalled(TrackPluginManagementJob), + PluginEnabled(TrackPluginManagementJob), + PluginDisabled(TrackPluginManagementJob), +} + +struct TrackSkillInvocationsJob { + analytics_enabled: Option, + tracking: TrackEventsContext, + invocations: Vec, +} + +struct TrackAppMentionedJob { + analytics_enabled: Option, + tracking: TrackEventsContext, + mentions: Vec, +} + +struct TrackAppUsedJob { + analytics_enabled: Option, + tracking: TrackEventsContext, + app: AppInvocation, +} + +struct TrackPluginUsedJob { + analytics_enabled: Option, + tracking: TrackEventsContext, + plugin: PluginTelemetryMetadata, +} + +struct TrackPluginManagementJob { + analytics_enabled: Option, + plugin: PluginTelemetryMetadata, +} + +#[derive(Clone, Copy)] +enum PluginManagementEventType { + Installed, + Uninstalled, + Enabled, + Disabled, +} + +const ANALYTICS_EVENTS_QUEUE_SIZE: usize = 256; +const ANALYTICS_EVENTS_TIMEOUT: Duration = Duration::from_secs(10); +const ANALYTICS_EVENT_DEDUPE_MAX_KEYS: usize = 4096; + +#[derive(Serialize)] +struct TrackEventsRequest { + events: Vec, +} + +#[derive(Serialize)] +#[serde(untagged)] +enum TrackEventRequest { + SkillInvocation(SkillInvocationEventRequest), + AppMentioned(CodexAppMentionedEventRequest), + AppUsed(CodexAppUsedEventRequest), + PluginUsed(CodexPluginUsedEventRequest), + PluginInstalled(CodexPluginEventRequest), + PluginUninstalled(CodexPluginEventRequest), + PluginEnabled(CodexPluginEventRequest), + PluginDisabled(CodexPluginEventRequest), +} + +#[derive(Serialize)] +struct SkillInvocationEventRequest { + event_type: &'static str, + skill_id: String, + skill_name: String, + event_params: SkillInvocationEventParams, +} + +#[derive(Serialize)] +struct SkillInvocationEventParams { + product_client_id: Option, + skill_scope: Option, + repo_url: Option, + thread_id: Option, + invoke_type: Option, + model_slug: Option, +} + +#[derive(Serialize)] +struct CodexAppMetadata { + connector_id: Option, + thread_id: Option, + turn_id: Option, + app_name: Option, + product_client_id: Option, + invoke_type: Option, + model_slug: Option, +} + +#[derive(Serialize)] +struct CodexAppMentionedEventRequest { + event_type: &'static str, + event_params: CodexAppMetadata, +} + +#[derive(Serialize)] +struct CodexAppUsedEventRequest { + event_type: &'static str, + event_params: CodexAppMetadata, +} + +#[derive(Serialize)] +struct CodexPluginMetadata { + plugin_id: Option, + plugin_name: Option, + marketplace_name: Option, + has_skills: Option, + mcp_server_count: Option, + connector_ids: Option>, + product_client_id: Option, +} + +#[derive(Serialize)] +struct CodexPluginUsedMetadata { + #[serde(flatten)] + plugin: CodexPluginMetadata, + thread_id: Option, + turn_id: Option, + model_slug: Option, +} + +#[derive(Serialize)] +struct CodexPluginEventRequest { + event_type: &'static str, + event_params: CodexPluginMetadata, +} + +#[derive(Serialize)] +struct CodexPluginUsedEventRequest { + event_type: &'static str, + event_params: CodexPluginUsedMetadata, +} + +pub(crate) fn track_skill_invocations( + queue: &AnalyticsEventsQueue, + analytics_enabled: Option, + tracking: Option, + invocations: Vec, +) { + if analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if invocations.is_empty() { + return; + } + let job = TrackEventsJob::SkillInvocations(TrackSkillInvocationsJob { + analytics_enabled, + tracking, + invocations, + }); + queue.try_send(job); +} + +pub(crate) fn track_app_mentioned( + queue: &AnalyticsEventsQueue, + analytics_enabled: Option, + tracking: Option, + mentions: Vec, +) { + if analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if mentions.is_empty() { + return; + } + let job = TrackEventsJob::AppMentioned(TrackAppMentionedJob { + analytics_enabled, + tracking, + mentions, + }); + queue.try_send(job); +} + +pub(crate) fn track_app_used( + queue: &AnalyticsEventsQueue, + analytics_enabled: Option, + tracking: Option, + app: AppInvocation, +) { + if analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if !queue.should_enqueue_app_used(&tracking, &app) { + return; + } + let job = TrackEventsJob::AppUsed(TrackAppUsedJob { + analytics_enabled, + tracking, + app, + }); + queue.try_send(job); +} + +pub(crate) fn track_plugin_used( + queue: &AnalyticsEventsQueue, + analytics_enabled: Option, + tracking: Option, + plugin: PluginTelemetryMetadata, +) { + if analytics_enabled == Some(false) { + return; + } + let Some(tracking) = tracking else { + return; + }; + if !queue.should_enqueue_plugin_used(&tracking, &plugin) { + return; + } + let job = TrackEventsJob::PluginUsed(TrackPluginUsedJob { + analytics_enabled, + tracking, + plugin, + }); + queue.try_send(job); +} + +fn track_plugin_management( + queue: &AnalyticsEventsQueue, + analytics_enabled: Option, + event_type: PluginManagementEventType, + plugin: PluginTelemetryMetadata, +) { + if analytics_enabled == Some(false) { + return; + } + let job = TrackPluginManagementJob { + analytics_enabled, + plugin, + }; + let job = match event_type { + PluginManagementEventType::Installed => TrackEventsJob::PluginInstalled(job), + PluginManagementEventType::Uninstalled => TrackEventsJob::PluginUninstalled(job), + PluginManagementEventType::Enabled => TrackEventsJob::PluginEnabled(job), + PluginManagementEventType::Disabled => TrackEventsJob::PluginDisabled(job), + }; + queue.try_send(job); +} + +async fn send_track_skill_invocations( + auth_manager: &AuthManager, + base_url: &str, + job: TrackSkillInvocationsJob, +) { + let TrackSkillInvocationsJob { + analytics_enabled, + tracking, + invocations, + } = job; + let mut events = Vec::with_capacity(invocations.len()); + for invocation in invocations { + let skill_scope = match invocation.skill_scope { + SkillScope::User => "user", + SkillScope::Repo => "repo", + SkillScope::System => "system", + SkillScope::Admin => "admin", + }; + let repo_root = get_git_repo_root(invocation.skill_path.as_path()); + let repo_url = if let Some(root) = repo_root.as_ref() { + collect_git_info(root) + .await + .and_then(|info| info.repository_url) + } else { + None + }; + let skill_id = skill_id_for_local_skill( + repo_url.as_deref(), + repo_root.as_deref(), + invocation.skill_path.as_path(), + invocation.skill_name.as_str(), + ); + events.push(TrackEventRequest::SkillInvocation( + SkillInvocationEventRequest { + event_type: "skill_invocation", + skill_id, + skill_name: invocation.skill_name.clone(), + event_params: SkillInvocationEventParams { + thread_id: Some(tracking.thread_id.clone()), + invoke_type: Some(invocation.invocation_type), + model_slug: Some(tracking.model_slug.clone()), + product_client_id: Some(originator().value), + repo_url, + skill_scope: Some(skill_scope.to_string()), + }, + }, + )); + } + + send_track_events(auth_manager, analytics_enabled, base_url, events).await; +} + +async fn send_track_app_mentioned( + auth_manager: &AuthManager, + base_url: &str, + job: TrackAppMentionedJob, +) { + let TrackAppMentionedJob { + analytics_enabled, + tracking, + mentions, + } = job; + let events = mentions + .into_iter() + .map(|mention| { + let event_params = codex_app_metadata(&tracking, mention); + TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest { + event_type: "codex_app_mentioned", + event_params, + }) + }) + .collect::>(); + + send_track_events(auth_manager, analytics_enabled, base_url, events).await; +} + +async fn send_track_app_used(auth_manager: &AuthManager, base_url: &str, job: TrackAppUsedJob) { + let TrackAppUsedJob { + analytics_enabled, + tracking, + app, + } = job; + let event_params = codex_app_metadata(&tracking, app); + let events = vec![TrackEventRequest::AppUsed(CodexAppUsedEventRequest { + event_type: "codex_app_used", + event_params, + })]; + + send_track_events(auth_manager, analytics_enabled, base_url, events).await; +} + +async fn send_track_plugin_used( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginUsedJob, +) { + let TrackPluginUsedJob { + analytics_enabled, + tracking, + plugin, + } = job; + let events = vec![TrackEventRequest::PluginUsed(CodexPluginUsedEventRequest { + event_type: "codex_plugin_used", + event_params: codex_plugin_used_metadata(&tracking, plugin), + })]; + + send_track_events(auth_manager, analytics_enabled, base_url, events).await; +} + +async fn send_track_plugin_installed( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginManagementJob, +) { + send_track_plugin_management_event(auth_manager, base_url, job, "codex_plugin_installed").await; +} + +async fn send_track_plugin_uninstalled( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginManagementJob, +) { + send_track_plugin_management_event(auth_manager, base_url, job, "codex_plugin_uninstalled") + .await; +} + +async fn send_track_plugin_enabled( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginManagementJob, +) { + send_track_plugin_management_event(auth_manager, base_url, job, "codex_plugin_enabled").await; +} + +async fn send_track_plugin_disabled( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginManagementJob, +) { + send_track_plugin_management_event(auth_manager, base_url, job, "codex_plugin_disabled").await; +} + +async fn send_track_plugin_management_event( + auth_manager: &AuthManager, + base_url: &str, + job: TrackPluginManagementJob, + event_type: &'static str, +) { + let TrackPluginManagementJob { + analytics_enabled, + plugin, + } = job; + let event_params = codex_plugin_metadata(plugin); + let event = CodexPluginEventRequest { + event_type, + event_params, + }; + let events = vec![match event_type { + "codex_plugin_installed" => TrackEventRequest::PluginInstalled(event), + "codex_plugin_uninstalled" => TrackEventRequest::PluginUninstalled(event), + "codex_plugin_enabled" => TrackEventRequest::PluginEnabled(event), + "codex_plugin_disabled" => TrackEventRequest::PluginDisabled(event), + _ => unreachable!("unknown plugin management event type"), + }]; + + send_track_events(auth_manager, analytics_enabled, base_url, events).await; +} + +fn codex_app_metadata(tracking: &TrackEventsContext, app: AppInvocation) -> CodexAppMetadata { + CodexAppMetadata { + connector_id: app.connector_id, + thread_id: Some(tracking.thread_id.clone()), + turn_id: Some(tracking.turn_id.clone()), + app_name: app.app_name, + product_client_id: Some(originator().value), + invoke_type: app.invocation_type, + model_slug: Some(tracking.model_slug.clone()), + } +} + +fn codex_plugin_metadata(plugin: PluginTelemetryMetadata) -> CodexPluginMetadata { + let capability_summary = plugin.capability_summary; + CodexPluginMetadata { + plugin_id: Some(plugin.plugin_id.as_key()), + plugin_name: Some(plugin.plugin_id.plugin_name), + marketplace_name: Some(plugin.plugin_id.marketplace_name), + has_skills: capability_summary + .as_ref() + .map(|summary| summary.has_skills), + mcp_server_count: capability_summary + .as_ref() + .map(|summary| summary.mcp_server_names.len()), + connector_ids: capability_summary.map(|summary| { + summary + .app_connector_ids + .into_iter() + .map(|connector_id| connector_id.0) + .collect() + }), + product_client_id: Some(originator().value), + } +} + +fn codex_plugin_used_metadata( + tracking: &TrackEventsContext, + plugin: PluginTelemetryMetadata, +) -> CodexPluginUsedMetadata { + CodexPluginUsedMetadata { + plugin: codex_plugin_metadata(plugin), + thread_id: Some(tracking.thread_id.clone()), + turn_id: Some(tracking.turn_id.clone()), + model_slug: Some(tracking.model_slug.clone()), + } +} + +async fn send_track_events( + auth_manager: &AuthManager, + analytics_enabled: Option, + base_url: &str, + events: Vec, +) { + if analytics_enabled == Some(false) { + return; + } + if events.is_empty() { + return; + } + let Some(auth) = auth_manager.auth().await else { + return; + }; + if !auth.is_chatgpt_auth() { + return; + } + let access_token = match auth.get_token() { + Ok(token) => token, + Err(_) => return, + }; + let Some(account_id) = auth.get_account_id() else { + return; + }; + + let base_url = base_url.trim_end_matches('/'); + let url = format!("{base_url}/codex/analytics-events/events"); + let payload = TrackEventsRequest { events }; + + let response = create_client() + .post(&url) + .timeout(ANALYTICS_EVENTS_TIMEOUT) + .bearer_auth(&access_token) + .header("chatgpt-account-id", &account_id) + .header("Content-Type", "application/json") + .json(&payload) + .send() + .await; + + match response { + Ok(response) if response.status().is_success() => {} + Ok(response) => { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + tracing::warn!("events failed with status {status}: {body}"); + } + Err(err) => { + tracing::warn!("failed to send events request: {err}"); + } + } +} + +pub(crate) fn skill_id_for_local_skill( + repo_url: Option<&str>, + repo_root: Option<&Path>, + skill_path: &Path, + skill_name: &str, +) -> String { + let path = normalize_path_for_skill_id(repo_url, repo_root, skill_path); + let prefix = if let Some(url) = repo_url { + format!("repo_{url}") + } else { + "personal".to_string() + }; + let raw_id = format!("{prefix}_{path}_{skill_name}"); + let mut hasher = Sha1::new(); + hasher.update(raw_id.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +/// Returns a normalized path for skill ID construction. +/// +/// - Repo-scoped skills use a path relative to the repo root. +/// - User/admin/system skills use an absolute path. +fn normalize_path_for_skill_id( + repo_url: Option<&str>, + repo_root: Option<&Path>, + skill_path: &Path, +) -> String { + let resolved_path = + std::fs::canonicalize(skill_path).unwrap_or_else(|_| skill_path.to_path_buf()); + match (repo_url, repo_root) { + (Some(_), Some(root)) => { + let resolved_root = std::fs::canonicalize(root).unwrap_or_else(|_| root.to_path_buf()); + resolved_path + .strip_prefix(&resolved_root) + .unwrap_or(resolved_path.as_path()) + .to_string_lossy() + .replace('\\', "/") + } + _ => resolved_path.to_string_lossy().replace('\\', "/"), + } +} + +#[cfg(test)] +#[path = "analytics_client_tests.rs"] +mod tests; diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs new file mode 100644 index 0000000000..f64da8f589 --- /dev/null +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -0,0 +1,290 @@ +use super::AnalyticsEventsQueue; +use super::AppInvocation; +use super::CodexAppMentionedEventRequest; +use super::CodexAppUsedEventRequest; +use super::CodexPluginEventRequest; +use super::CodexPluginUsedEventRequest; +use super::InvocationType; +use super::TrackEventRequest; +use super::TrackEventsContext; +use super::codex_app_metadata; +use super::codex_plugin_metadata; +use super::codex_plugin_used_metadata; +use super::normalize_path_for_skill_id; +use codex_login::default_client::originator; +use codex_plugin::AppConnectorId; +use codex_plugin::PluginCapabilitySummary; +use codex_plugin::PluginId; +use codex_plugin::PluginTelemetryMetadata; +use pretty_assertions::assert_eq; +use serde_json::json; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use tokio::sync::mpsc; + +fn expected_absolute_path(path: &PathBuf) -> String { + std::fs::canonicalize(path) + .unwrap_or_else(|_| path.to_path_buf()) + .to_string_lossy() + .replace('\\', "/") +} + +#[test] +fn normalize_path_for_skill_id_repo_scoped_uses_relative_path() { + let repo_root = PathBuf::from("/repo/root"); + let skill_path = PathBuf::from("/repo/root/.codex/skills/doc/SKILL.md"); + + let path = normalize_path_for_skill_id( + Some("https://example.com/repo.git"), + Some(repo_root.as_path()), + skill_path.as_path(), + ); + + assert_eq!(path, ".codex/skills/doc/SKILL.md"); +} + +#[test] +fn normalize_path_for_skill_id_user_scoped_uses_absolute_path() { + let skill_path = PathBuf::from("/Users/abc/.codex/skills/doc/SKILL.md"); + + let path = normalize_path_for_skill_id(None, None, skill_path.as_path()); + let expected = expected_absolute_path(&skill_path); + + assert_eq!(path, expected); +} + +#[test] +fn normalize_path_for_skill_id_admin_scoped_uses_absolute_path() { + let skill_path = PathBuf::from("/etc/codex/skills/doc/SKILL.md"); + + let path = normalize_path_for_skill_id(None, None, skill_path.as_path()); + let expected = expected_absolute_path(&skill_path); + + assert_eq!(path, expected); +} + +#[test] +fn normalize_path_for_skill_id_repo_root_not_in_skill_path_uses_absolute_path() { + let repo_root = PathBuf::from("/repo/root"); + let skill_path = PathBuf::from("/other/path/.codex/skills/doc/SKILL.md"); + + let path = normalize_path_for_skill_id( + Some("https://example.com/repo.git"), + Some(repo_root.as_path()), + skill_path.as_path(), + ); + let expected = expected_absolute_path(&skill_path); + + assert_eq!(path, expected); +} + +#[test] +fn app_mentioned_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + let event = TrackEventRequest::AppMentioned(CodexAppMentionedEventRequest { + event_type: "codex_app_mentioned", + event_params: codex_app_metadata( + &tracking, + AppInvocation { + connector_id: Some("calendar".to_string()), + app_name: Some("Calendar".to_string()), + invocation_type: Some(InvocationType::Explicit), + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize app mentioned event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_app_mentioned", + "event_params": { + "connector_id": "calendar", + "thread_id": "thread-1", + "turn_id": "turn-1", + "app_name": "Calendar", + "product_client_id": originator().value, + "invoke_type": "explicit", + "model_slug": "gpt-5" + } + }) + ); +} + +#[test] +fn app_used_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + }; + let event = TrackEventRequest::AppUsed(CodexAppUsedEventRequest { + event_type: "codex_app_used", + event_params: codex_app_metadata( + &tracking, + AppInvocation { + connector_id: Some("drive".to_string()), + app_name: Some("Google Drive".to_string()), + invocation_type: Some(InvocationType::Implicit), + }, + ), + }); + + let payload = serde_json::to_value(&event).expect("serialize app used event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_app_used", + "event_params": { + "connector_id": "drive", + "thread_id": "thread-2", + "turn_id": "turn-2", + "app_name": "Google Drive", + "product_client_id": originator().value, + "invoke_type": "implicit", + "model_slug": "gpt-5" + } + }) + ); +} + +#[test] +fn app_used_dedupe_is_keyed_by_turn_and_connector() { + let (sender, _receiver) = mpsc::channel(1); + let queue = AnalyticsEventsQueue { + sender, + app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + plugin_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + }; + let app = AppInvocation { + connector_id: Some("calendar".to_string()), + app_name: Some("Calendar".to_string()), + invocation_type: Some(InvocationType::Implicit), + }; + + let turn_1 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + let turn_2 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-2".to_string(), + }; + + assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), true); + assert_eq!(queue.should_enqueue_app_used(&turn_1, &app), false); + assert_eq!(queue.should_enqueue_app_used(&turn_2, &app), true); +} + +#[test] +fn plugin_used_event_serializes_expected_shape() { + let tracking = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-3".to_string(), + turn_id: "turn-3".to_string(), + }; + let event = TrackEventRequest::PluginUsed(CodexPluginUsedEventRequest { + event_type: "codex_plugin_used", + event_params: codex_plugin_used_metadata(&tracking, sample_plugin_metadata()), + }); + + let payload = serde_json::to_value(&event).expect("serialize plugin used event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_plugin_used", + "event_params": { + "plugin_id": "sample@test", + "plugin_name": "sample", + "marketplace_name": "test", + "has_skills": true, + "mcp_server_count": 2, + "connector_ids": ["calendar", "drive"], + "product_client_id": originator().value, + "thread_id": "thread-3", + "turn_id": "turn-3", + "model_slug": "gpt-5" + } + }) + ); +} + +#[test] +fn plugin_management_event_serializes_expected_shape() { + let event = TrackEventRequest::PluginInstalled(CodexPluginEventRequest { + event_type: "codex_plugin_installed", + event_params: codex_plugin_metadata(sample_plugin_metadata()), + }); + + let payload = serde_json::to_value(&event).expect("serialize plugin installed event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_plugin_installed", + "event_params": { + "plugin_id": "sample@test", + "plugin_name": "sample", + "marketplace_name": "test", + "has_skills": true, + "mcp_server_count": 2, + "connector_ids": ["calendar", "drive"], + "product_client_id": originator().value + } + }) + ); +} + +#[test] +fn plugin_used_dedupe_is_keyed_by_turn_and_plugin() { + let (sender, _receiver) = mpsc::channel(1); + let queue = AnalyticsEventsQueue { + sender, + app_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + plugin_used_emitted_keys: Arc::new(Mutex::new(HashSet::new())), + }; + let plugin = sample_plugin_metadata(); + + let turn_1 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }; + let turn_2 = TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-2".to_string(), + }; + + assert_eq!(queue.should_enqueue_plugin_used(&turn_1, &plugin), true); + assert_eq!(queue.should_enqueue_plugin_used(&turn_1, &plugin), false); + assert_eq!(queue.should_enqueue_plugin_used(&turn_2, &plugin), true); +} + +fn sample_plugin_metadata() -> PluginTelemetryMetadata { + PluginTelemetryMetadata { + plugin_id: PluginId::parse("sample@test").expect("valid plugin id"), + capability_summary: Some(PluginCapabilitySummary { + config_name: "sample@test".to_string(), + display_name: "sample".to_string(), + description: None, + has_skills: true, + mcp_server_names: vec!["mcp-1".to_string(), "mcp-2".to_string()], + app_connector_ids: vec![ + AppConnectorId("calendar".to_string()), + AppConnectorId("drive".to_string()), + ], + }), + } +} diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs new file mode 100644 index 0000000000..e36f5d71b5 --- /dev/null +++ b/codex-rs/analytics/src/lib.rs @@ -0,0 +1,8 @@ +mod analytics_client; + +pub use analytics_client::AnalyticsEventsClient; +pub use analytics_client::AppInvocation; +pub use analytics_client::InvocationType; +pub use analytics_client::SkillInvocation; +pub use analytics_client::TrackEventsContext; +pub use analytics_client::build_track_events_context; diff --git a/codex-rs/config/src/project_root_markers.rs b/codex-rs/config/src/project_root_markers.rs new file mode 100644 index 0000000000..3061dacc6b --- /dev/null +++ b/codex-rs/config/src/project_root_markers.rs @@ -0,0 +1,50 @@ +use std::io; + +use toml::Value as TomlValue; + +const DEFAULT_PROJECT_ROOT_MARKERS: &[&str] = &[".git"]; + +/// Reads `project_root_markers` from a merged `config.toml` [toml::Value]. +/// +/// Invariants: +/// - If `project_root_markers` is not specified, returns `Ok(None)`. +/// - If `project_root_markers` is specified, returns `Ok(Some(markers))` where +/// `markers` is a `Vec` (including `Ok(Some(Vec::new()))` for an +/// empty array, which indicates that root detection should be disabled). +/// - Returns an error if `project_root_markers` is specified but is not an +/// array of strings. +pub fn project_root_markers_from_config(config: &TomlValue) -> io::Result>> { + let Some(table) = config.as_table() else { + return Ok(None); + }; + let Some(markers_value) = table.get("project_root_markers") else { + return Ok(None); + }; + let TomlValue::Array(entries) = markers_value else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "project_root_markers must be an array of strings", + )); + }; + if entries.is_empty() { + return Ok(Some(Vec::new())); + } + let mut markers = Vec::new(); + for entry in entries { + let Some(marker) = entry.as_str() else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "project_root_markers must be an array of strings", + )); + }; + markers.push(marker.to_string()); + } + Ok(Some(markers)) +} + +pub fn default_project_root_markers() -> Vec { + DEFAULT_PROJECT_ROOT_MARKERS + .iter() + .map(ToString::to_string) + .collect() +} diff --git a/codex-rs/core-skills/src/skills/mention_counts.rs b/codex-rs/core-skills/src/skills/mention_counts.rs new file mode 100644 index 0000000000..a9b3da9d30 --- /dev/null +++ b/codex-rs/core-skills/src/skills/mention_counts.rs @@ -0,0 +1,24 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +use super::SkillMetadata; + +/// Counts how often each skill name appears (exact and ASCII-lowercase), excluding disabled paths. +pub fn build_skill_name_counts( + skills: &[SkillMetadata], + disabled_paths: &HashSet, +) -> (HashMap, HashMap) { + let mut exact_counts: HashMap = HashMap::new(); + let mut lower_counts: HashMap = HashMap::new(); + for skill in skills { + if disabled_paths.contains(&skill.path_to_skills_md) { + continue; + } + *exact_counts.entry(skill.name.clone()).or_insert(0) += 1; + *lower_counts + .entry(skill.name.to_ascii_lowercase()) + .or_insert(0) += 1; + } + (exact_counts, lower_counts) +} diff --git a/codex-rs/core/src/analytics_client.rs.bak b/codex-rs/core/src/analytics_client.rs.bak new file mode 100644 index 0000000000..e69de29bb2 diff --git a/codex-rs/core/src/plugins/mentions.rs b/codex-rs/core/src/plugins/mentions.rs new file mode 100644 index 0000000000..37c8680261 --- /dev/null +++ b/codex-rs/core/src/plugins/mentions.rs @@ -0,0 +1,121 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +use codex_protocol::user_input::UserInput; + +use crate::connectors; +use crate::mention_syntax::PLUGIN_TEXT_MENTION_SIGIL; +use crate::mention_syntax::TOOL_MENTION_SIGIL; +use crate::skills::SkillMetadata; +use crate::skills::injection::ToolMentionKind; +use crate::skills::injection::app_id_from_path; +use crate::skills::injection::extract_tool_mentions_with_sigil; +use crate::skills::injection::plugin_config_name_from_path; +use crate::skills::injection::tool_kind_for_path; + +use super::PluginCapabilitySummary; + +pub(crate) struct CollectedToolMentions { + pub(crate) plain_names: HashSet, + pub(crate) paths: HashSet, +} + +pub(crate) fn collect_tool_mentions_from_messages(messages: &[String]) -> CollectedToolMentions { + collect_tool_mentions_from_messages_with_sigil(messages, TOOL_MENTION_SIGIL) +} + +fn collect_tool_mentions_from_messages_with_sigil( + messages: &[String], + sigil: char, +) -> CollectedToolMentions { + let mut plain_names = HashSet::new(); + let mut paths = HashSet::new(); + for message in messages { + let mentions = extract_tool_mentions_with_sigil(message, sigil); + plain_names.extend(mentions.plain_names().map(str::to_string)); + paths.extend(mentions.paths().map(str::to_string)); + } + CollectedToolMentions { plain_names, paths } +} + +pub(crate) fn collect_explicit_app_ids(input: &[UserInput]) -> HashSet { + let messages = input + .iter() + .filter_map(|item| match item { + UserInput::Text { text, .. } => Some(text.clone()), + _ => None, + }) + .collect::>(); + + input + .iter() + .filter_map(|item| match item { + UserInput::Mention { path, .. } => Some(path.clone()), + _ => None, + }) + .chain(collect_tool_mentions_from_messages(&messages).paths) + .filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::App) + .filter_map(|path| app_id_from_path(path.as_str()).map(str::to_string)) + .collect() +} + +/// Collect explicit structured or linked `plugin://...` mentions. +pub(crate) fn collect_explicit_plugin_mentions( + input: &[UserInput], + plugins: &[PluginCapabilitySummary], +) -> Vec { + if plugins.is_empty() { + return Vec::new(); + } + + let messages = input + .iter() + .filter_map(|item| match item { + UserInput::Text { text, .. } => Some(text.clone()), + _ => None, + }) + .collect::>(); + + let mentioned_config_names: HashSet = input + .iter() + .filter_map(|item| match item { + UserInput::Mention { path, .. } => Some(path.clone()), + _ => None, + }) + .chain( + // Plugin plaintext links use `@`, not the default `$` tool sigil. + collect_tool_mentions_from_messages_with_sigil(&messages, PLUGIN_TEXT_MENTION_SIGIL) + .paths, + ) + .filter(|path| tool_kind_for_path(path.as_str()) == ToolMentionKind::Plugin) + .filter_map(|path| plugin_config_name_from_path(path.as_str()).map(str::to_string)) + .collect(); + + if mentioned_config_names.is_empty() { + return Vec::new(); + } + + plugins + .iter() + .filter(|plugin| mentioned_config_names.contains(plugin.config_name.as_str())) + .cloned() + .collect() +} + +pub(crate) use crate::skills::build_skill_name_counts; + +pub(crate) fn build_connector_slug_counts( + connectors: &[connectors::AppInfo], +) -> HashMap { + let mut counts: HashMap = HashMap::new(); + for connector in connectors { + let slug = connectors::connector_mention_slug(connector); + *counts.entry(slug).or_insert(0) += 1; + } + counts +} + +#[cfg(test)] +#[path = "mentions_tests.rs"] +mod tests; diff --git a/codex-rs/core/src/plugins/mentions_tests.rs b/codex-rs/core/src/plugins/mentions_tests.rs new file mode 100644 index 0000000000..37c9adb886 --- /dev/null +++ b/codex-rs/core/src/plugins/mentions_tests.rs @@ -0,0 +1,155 @@ +use std::collections::HashSet; + +use codex_protocol::user_input::UserInput; +use pretty_assertions::assert_eq; + +use super::collect_explicit_app_ids; +use super::collect_explicit_plugin_mentions; +use crate::plugins::PluginCapabilitySummary; + +fn text_input(text: &str) -> UserInput { + UserInput::Text { + text: text.to_string(), + text_elements: Vec::new(), + } +} + +fn plugin(config_name: &str, display_name: &str) -> PluginCapabilitySummary { + PluginCapabilitySummary { + config_name: config_name.to_string(), + display_name: display_name.to_string(), + description: None, + has_skills: true, + mcp_server_names: Vec::new(), + app_connector_ids: Vec::new(), + } +} + +#[test] +fn collect_explicit_app_ids_from_linked_text_mentions() { + let input = vec![text_input("use [$calendar](app://calendar)")]; + + let app_ids = collect_explicit_app_ids(&input); + + assert_eq!(app_ids, HashSet::from(["calendar".to_string()])); +} + +#[test] +fn collect_explicit_app_ids_dedupes_structured_and_linked_mentions() { + let input = vec![ + text_input("use [$calendar](app://calendar)"), + UserInput::Mention { + name: "calendar".to_string(), + path: "app://calendar".to_string(), + }, + ]; + + let app_ids = collect_explicit_app_ids(&input); + + assert_eq!(app_ids, HashSet::from(["calendar".to_string()])); +} + +#[test] +fn collect_explicit_app_ids_ignores_non_app_paths() { + let input = vec![ + text_input( + "use [$docs](mcp://docs) and [$skill](skill://team/skill) and [$file](/tmp/file.txt)", + ), + UserInput::Mention { + name: "docs".to_string(), + path: "mcp://docs".to_string(), + }, + UserInput::Mention { + name: "skill".to_string(), + path: "skill://team/skill".to_string(), + }, + UserInput::Mention { + name: "file".to_string(), + path: "/tmp/file.txt".to_string(), + }, + ]; + + let app_ids = collect_explicit_app_ids(&input); + + assert_eq!(app_ids, HashSet::::new()); +} + +#[test] +fn collect_explicit_plugin_mentions_from_structured_paths() { + let plugins = vec![ + plugin("sample@test", "sample"), + plugin("other@test", "other"), + ]; + + let mentioned = collect_explicit_plugin_mentions( + &[UserInput::Mention { + name: "sample".to_string(), + path: "plugin://sample@test".to_string(), + }], + &plugins, + ); + + assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); +} + +#[test] +fn collect_explicit_plugin_mentions_from_linked_text_mentions() { + let plugins = vec![ + plugin("sample@test", "sample"), + plugin("other@test", "other"), + ]; + + let mentioned = collect_explicit_plugin_mentions( + &[text_input("use [@sample](plugin://sample@test)")], + &plugins, + ); + + assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); +} + +#[test] +fn collect_explicit_plugin_mentions_dedupes_structured_and_linked_mentions() { + let plugins = vec![ + plugin("sample@test", "sample"), + plugin("other@test", "other"), + ]; + + let mentioned = collect_explicit_plugin_mentions( + &[ + text_input("use [@sample](plugin://sample@test)"), + UserInput::Mention { + name: "sample".to_string(), + path: "plugin://sample@test".to_string(), + }, + ], + &plugins, + ); + + assert_eq!(mentioned, vec![plugin("sample@test", "sample")]); +} + +#[test] +fn collect_explicit_plugin_mentions_ignores_non_plugin_paths() { + let plugins = vec![plugin("sample@test", "sample")]; + + let mentioned = collect_explicit_plugin_mentions( + &[text_input( + "use [$app](app://calendar) and [$skill](skill://team/skill) and [$file](/tmp/file.txt)", + )], + &plugins, + ); + + assert_eq!(mentioned, Vec::::new()); +} + +#[test] +fn collect_explicit_plugin_mentions_ignores_dollar_linked_plugin_mentions() { + let plugins = vec![plugin("sample@test", "sample")]; + + let mentioned = collect_explicit_plugin_mentions( + &[text_input("use [$sample](plugin://sample@test)")], + &plugins, + ); + + assert_eq!(mentioned, Vec::::new()); +} diff --git a/codex-rs/instructions/BUILD.bazel b/codex-rs/instructions/BUILD.bazel new file mode 100644 index 0000000000..d3cdbd19d1 --- /dev/null +++ b/codex-rs/instructions/BUILD.bazel @@ -0,0 +1,16 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "instructions", + crate_name = "codex_instructions", + compile_data = glob( + include = ["**"], + exclude = [ + "BUILD.bazel", + "Cargo.toml", + ], + allow_empty = True, + ) + [ + "//codex-rs:node-version.txt", + ], +) diff --git a/codex-rs/instructions/Cargo.toml b/codex-rs/instructions/Cargo.toml new file mode 100644 index 0000000000..cdaa3d7368 --- /dev/null +++ b/codex-rs/instructions/Cargo.toml @@ -0,0 +1,20 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-instructions" +version.workspace = true + +[lib] +doctest = false +name = "codex_instructions" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +codex-protocol = { workspace = true } +serde = { workspace = true, features = ["derive"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/codex-rs/instructions/src/fragment.rs b/codex-rs/instructions/src/fragment.rs new file mode 100644 index 0000000000..27509076fe --- /dev/null +++ b/codex-rs/instructions/src/fragment.rs @@ -0,0 +1,61 @@ +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; + +pub const AGENTS_MD_START_MARKER: &str = "# AGENTS.md instructions for "; +pub const AGENTS_MD_END_MARKER: &str = ""; +pub const SKILL_OPEN_TAG: &str = ""; +pub const SKILL_CLOSE_TAG: &str = ""; + +#[derive(Clone, Copy)] +pub struct ContextualUserFragmentDefinition { + start_marker: &'static str, + end_marker: &'static str, +} + +impl ContextualUserFragmentDefinition { + pub const fn new(start_marker: &'static str, end_marker: &'static str) -> Self { + Self { + start_marker, + end_marker, + } + } + + pub fn matches_text(&self, text: &str) -> bool { + let trimmed = text.trim_start(); + let starts_with_marker = trimmed + .get(..self.start_marker.len()) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(self.start_marker)); + let trimmed = trimmed.trim_end(); + let ends_with_marker = trimmed + .get(trimmed.len().saturating_sub(self.end_marker.len())..) + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(self.end_marker)); + starts_with_marker && ends_with_marker + } + + pub const fn start_marker(&self) -> &'static str { + self.start_marker + } + + pub const fn end_marker(&self) -> &'static str { + self.end_marker + } + + pub fn wrap(&self, body: String) -> String { + format!("{}\n{}\n{}", self.start_marker, body, self.end_marker) + } + + pub fn into_message(self, text: String) -> ResponseItem { + ResponseItem::Message { + id: None, + role: "user".to_string(), + content: vec![ContentItem::InputText { text }], + end_turn: None, + phase: None, + } + } +} + +pub const AGENTS_MD_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new(AGENTS_MD_START_MARKER, AGENTS_MD_END_MARKER); +pub const SKILL_FRAGMENT: ContextualUserFragmentDefinition = + ContextualUserFragmentDefinition::new(SKILL_OPEN_TAG, SKILL_CLOSE_TAG); diff --git a/codex-rs/instructions/src/lib.rs b/codex-rs/instructions/src/lib.rs new file mode 100644 index 0000000000..e513fe6e1c --- /dev/null +++ b/codex-rs/instructions/src/lib.rs @@ -0,0 +1,15 @@ +//! User and skill instruction payloads and contextual user fragment markers for Codex prompts. + +mod fragment; +mod user_instructions; + +pub use fragment::AGENTS_MD_END_MARKER; +pub use fragment::AGENTS_MD_FRAGMENT; +pub use fragment::AGENTS_MD_START_MARKER; +pub use fragment::ContextualUserFragmentDefinition; +pub use fragment::SKILL_CLOSE_TAG; +pub use fragment::SKILL_FRAGMENT; +pub use fragment::SKILL_OPEN_TAG; +pub use user_instructions::SkillInstructions; +pub use user_instructions::USER_INSTRUCTIONS_PREFIX; +pub use user_instructions::UserInstructions; diff --git a/codex-rs/instructions/src/user_instructions.rs b/codex-rs/instructions/src/user_instructions.rs new file mode 100644 index 0000000000..4fd266e766 --- /dev/null +++ b/codex-rs/instructions/src/user_instructions.rs @@ -0,0 +1,56 @@ +use serde::Deserialize; +use serde::Serialize; + +use codex_protocol::models::ResponseItem; + +use crate::fragment::AGENTS_MD_FRAGMENT; +use crate::fragment::AGENTS_MD_START_MARKER; +use crate::fragment::SKILL_FRAGMENT; + +pub const USER_INSTRUCTIONS_PREFIX: &str = AGENTS_MD_START_MARKER; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename = "user_instructions", rename_all = "snake_case")] +pub struct UserInstructions { + pub directory: String, + pub text: String, +} + +impl UserInstructions { + pub fn serialize_to_text(&self) -> String { + format!( + "{prefix}{directory}\n\n\n{contents}\n{suffix}", + prefix = AGENTS_MD_FRAGMENT.start_marker(), + directory = self.directory, + contents = self.text, + suffix = AGENTS_MD_FRAGMENT.end_marker(), + ) + } +} + +impl From for ResponseItem { + fn from(ui: UserInstructions) -> Self { + AGENTS_MD_FRAGMENT.into_message(ui.serialize_to_text()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename = "skill_instructions", rename_all = "snake_case")] +pub struct SkillInstructions { + pub name: String, + pub path: String, + pub contents: String, +} + +impl From for ResponseItem { + fn from(si: SkillInstructions) -> Self { + SKILL_FRAGMENT.into_message(SKILL_FRAGMENT.wrap(format!( + "{}\n{}\n{}", + si.name, si.path, si.contents + ))) + } +} + +#[cfg(test)] +#[path = "user_instructions_tests.rs"] +mod tests; diff --git a/codex-rs/instructions/src/user_instructions_tests.rs b/codex-rs/instructions/src/user_instructions_tests.rs new file mode 100644 index 0000000000..75f35d11b0 --- /dev/null +++ b/codex-rs/instructions/src/user_instructions_tests.rs @@ -0,0 +1,72 @@ +use super::*; +use codex_protocol::models::ContentItem; +use codex_protocol::models::ResponseItem; +use pretty_assertions::assert_eq; + +use crate::fragment::AGENTS_MD_FRAGMENT; +use crate::fragment::SKILL_FRAGMENT; + +#[test] +fn test_user_instructions() { + let user_instructions = UserInstructions { + directory: "test_directory".to_string(), + text: "test_text".to_string(), + }; + let response_item: ResponseItem = user_instructions.into(); + + let ResponseItem::Message { role, content, .. } = response_item else { + panic!("expected ResponseItem::Message"); + }; + + assert_eq!(role, "user"); + + let [ContentItem::InputText { text }] = content.as_slice() else { + panic!("expected one InputText content item"); + }; + + assert_eq!( + text, + "# AGENTS.md instructions for test_directory\n\n\ntest_text\n", + ); +} + +#[test] +fn test_is_user_instructions() { + assert!(AGENTS_MD_FRAGMENT.matches_text( + "# AGENTS.md instructions for test_directory\n\n\ntest_text\n" + )); + assert!(!AGENTS_MD_FRAGMENT.matches_text("test_text")); +} + +#[test] +fn test_skill_instructions() { + let skill_instructions = SkillInstructions { + name: "demo-skill".to_string(), + path: "skills/demo/SKILL.md".to_string(), + contents: "body".to_string(), + }; + let response_item: ResponseItem = skill_instructions.into(); + + let ResponseItem::Message { role, content, .. } = response_item else { + panic!("expected ResponseItem::Message"); + }; + + assert_eq!(role, "user"); + + let [ContentItem::InputText { text }] = content.as_slice() else { + panic!("expected one InputText content item"); + }; + + assert_eq!( + text, + "\ndemo-skill\nskills/demo/SKILL.md\nbody\n", + ); +} + +#[test] +fn test_is_skill_instructions() { + assert!(SKILL_FRAGMENT.matches_text( + "\ndemo-skill\nskills/demo/SKILL.md\nbody\n" + )); + assert!(!SKILL_FRAGMENT.matches_text("regular text")); +} diff --git a/codex-rs/plugin/src/mention_syntax.rs b/codex-rs/plugin/src/mention_syntax.rs new file mode 100644 index 0000000000..de5d1432b9 --- /dev/null +++ b/codex-rs/plugin/src/mention_syntax.rs @@ -0,0 +1,7 @@ +//! Sigils for tool/plugin mentions in plaintext (shared across Codex crates). + +/// Default plaintext sigil for tools. +pub const TOOL_MENTION_SIGIL: char = '$'; + +/// Plugins use `@` in linked plaintext outside TUI. +pub const PLUGIN_TEXT_MENTION_SIGIL: char = '@'; diff --git a/codex-rs/plugin/src/plugin_namespace.rs b/codex-rs/plugin/src/plugin_namespace.rs new file mode 100644 index 0000000000..6688ae0469 --- /dev/null +++ b/codex-rs/plugin/src/plugin_namespace.rs @@ -0,0 +1,70 @@ +//! Resolve plugin namespace from skill file paths by walking ancestors for `plugin.json`. + +use std::fs; +use std::path::Path; + +/// Relative path from a plugin root to its manifest file. +pub const PLUGIN_MANIFEST_PATH: &str = ".codex-plugin/plugin.json"; + +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawPluginManifestName { + #[serde(default)] + name: String, +} + +fn plugin_manifest_name(plugin_root: &Path) -> Option { + let manifest_path = plugin_root.join(PLUGIN_MANIFEST_PATH); + if !manifest_path.is_file() { + return None; + } + let contents = fs::read_to_string(&manifest_path).ok()?; + let RawPluginManifestName { name: raw_name } = serde_json::from_str(&contents).ok()?; + Some( + plugin_root + .file_name() + .and_then(|entry| entry.to_str()) + .filter(|_| raw_name.trim().is_empty()) + .unwrap_or(raw_name.as_str()) + .to_string(), + ) +} + +/// Returns the plugin manifest `name` for the nearest ancestor of `path` that contains a valid +/// plugin manifest (same `name` rules as full manifest loading in codex-core). +pub fn plugin_namespace_for_skill_path(path: &Path) -> Option { + for ancestor in path.ancestors() { + if let Some(name) = plugin_manifest_name(ancestor) { + return Some(name); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::plugin_namespace_for_skill_path; + use std::fs; + use tempfile::tempdir; + + #[test] + fn uses_manifest_name() { + let tmp = tempdir().expect("tempdir"); + let plugin_root = tmp.path().join("plugins/sample"); + let skill_path = plugin_root.join("skills/search/SKILL.md"); + + fs::create_dir_all(skill_path.parent().expect("parent")).expect("mkdir"); + fs::create_dir_all(plugin_root.join(".codex-plugin")).expect("mkdir manifest"); + fs::write( + plugin_root.join(".codex-plugin/plugin.json"), + r#"{"name":"sample"}"#, + ) + .expect("write manifest"); + fs::write(&skill_path, "---\ndescription: search\n---\n").expect("write skill"); + + assert_eq!( + plugin_namespace_for_skill_path(&skill_path), + Some("sample".to_string()) + ); + } +}