Preserve legacy plugin analytics identities

This commit is contained in:
James Watterson-Tenvooren
2026-06-11 14:01:43 -07:00
parent ad666f9697
commit e52a2b9afe
11 changed files with 228 additions and 56 deletions
@@ -131,6 +131,7 @@ use codex_login::default_client::originator;
use codex_plugin::AppConnectorId;
use codex_plugin::PluginCapabilitySummary;
use codex_plugin::PluginId;
use codex_plugin::PluginTelemetryLegacyIdSource;
use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::config_types::ApprovalsReviewer;
@@ -2877,6 +2878,7 @@ fn plugin_used_event_serializes_expected_shape() {
"event_type": "codex_plugin_used",
"event_params": {
"plugin_id": "sample@test",
"local_plugin_id": "sample@test",
"remote_plugin_id": null,
"plugin_name": "sample",
"marketplace_name": "test",
@@ -2908,6 +2910,7 @@ fn plugin_management_event_serializes_expected_shape() {
"event_type": "codex_plugin_installed",
"event_params": {
"plugin_id": "sample@test",
"local_plugin_id": "sample@test",
"remote_plugin_id": null,
"plugin_name": "sample",
"marketplace_name": "test",
@@ -2921,7 +2924,7 @@ fn plugin_management_event_serializes_expected_shape() {
}
#[test]
fn plugin_management_event_serializes_local_and_remote_plugin_ids() {
fn plugin_management_event_preserves_local_legacy_plugin_id() {
let mut plugin = sample_plugin_metadata();
plugin.remote_plugin_id = Some("plugins~Plugin_remote".to_string());
let event = TrackEventRequest::PluginInstalled(CodexPluginEventRequest {
@@ -2931,13 +2934,54 @@ fn plugin_management_event_serializes_local_and_remote_plugin_ids() {
let payload = serde_json::to_value(&event).expect("serialize plugin installed event");
assert_eq!(payload["event_params"]["plugin_id"], "sample@test");
assert_eq!(
payload["event_params"]["remote_plugin_id"],
"plugins~Plugin_remote"
payload,
json!({
"event_type": "codex_plugin_installed",
"event_params": {
"plugin_id": "sample@test",
"local_plugin_id": "sample@test",
"remote_plugin_id": "plugins~Plugin_remote",
"plugin_name": "sample",
"marketplace_name": "test",
"has_skills": true,
"mcp_server_count": 2,
"connector_ids": ["calendar", "drive"],
"product_client_id": originator().value
}
})
);
}
#[test]
fn plugin_management_event_preserves_remote_legacy_plugin_id() {
let mut plugin = sample_plugin_metadata();
plugin.remote_plugin_id = Some("plugins~Plugin_remote".to_string());
plugin.legacy_plugin_id_source = PluginTelemetryLegacyIdSource::Remote;
let event = TrackEventRequest::PluginInstalled(CodexPluginEventRequest {
event_type: "codex_plugin_installed",
event_params: codex_plugin_metadata(plugin),
});
let payload = serde_json::to_value(&event).expect("serialize plugin installed event");
assert_eq!(
payload,
json!({
"event_type": "codex_plugin_installed",
"event_params": {
"plugin_id": "plugins~Plugin_remote",
"local_plugin_id": "sample@test",
"remote_plugin_id": "plugins~Plugin_remote",
"plugin_name": "sample",
"marketplace_name": "test",
"has_skills": true,
"mcp_server_count": 2,
"connector_ids": ["calendar", "drive"],
"product_client_id": originator().value
}
})
);
assert_eq!(payload["event_params"]["plugin_name"], "sample");
assert_eq!(payload["event_params"]["marketplace_name"], "test");
}
#[test]
@@ -3277,6 +3321,7 @@ async fn reducer_ingests_plugin_state_changed_fact() {
"event_type": "codex_plugin_disabled",
"event_params": {
"plugin_id": "sample@test",
"local_plugin_id": "sample@test",
"remote_plugin_id": null,
"plugin_name": "sample",
"marketplace_name": "test",
@@ -4188,6 +4233,7 @@ fn sample_plugin_metadata() -> PluginTelemetryMetadata {
PluginTelemetryMetadata {
plugin_id: PluginId::parse("sample@test").expect("valid plugin id"),
remote_plugin_id: None,
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "sample".to_string(),
+12 -1
View File
@@ -24,6 +24,7 @@ use crate::now_unix_millis;
use codex_app_server_protocol::CodexErrorInfo;
use codex_app_server_protocol::CommandExecutionSource;
use codex_login::default_client::originator;
use codex_plugin::PluginTelemetryLegacyIdSource;
use codex_plugin::PluginTelemetryMetadata;
use codex_protocol::approvals::NetworkApprovalProtocol;
use codex_protocol::models::AdditionalPermissionProfile;
@@ -863,6 +864,7 @@ pub(crate) struct CodexTurnSteerEventRequest {
#[derive(Serialize)]
pub(crate) struct CodexPluginMetadata {
pub(crate) plugin_id: Option<String>,
pub(crate) local_plugin_id: Option<String>,
pub(crate) remote_plugin_id: Option<String>,
pub(crate) plugin_name: Option<String>,
pub(crate) marketplace_name: Option<String>,
@@ -922,10 +924,19 @@ pub(crate) fn codex_plugin_metadata(plugin: PluginTelemetryMetadata) -> CodexPlu
let PluginTelemetryMetadata {
plugin_id,
remote_plugin_id,
legacy_plugin_id_source,
capability_summary,
} = plugin;
let local_plugin_id = plugin_id.as_key();
let legacy_plugin_id = match legacy_plugin_id_source {
PluginTelemetryLegacyIdSource::Local => local_plugin_id.clone(),
PluginTelemetryLegacyIdSource::Remote => remote_plugin_id
.clone()
.unwrap_or_else(|| local_plugin_id.clone()),
};
CodexPluginMetadata {
plugin_id: Some(plugin_id.as_key()),
plugin_id: Some(legacy_plugin_id),
local_plugin_id: Some(local_plugin_id),
remote_plugin_id,
plugin_name: Some(plugin_id.plugin_name),
marketplace_name: Some(plugin_id.marketplace_name),
+8 -2
View File
@@ -42,8 +42,10 @@ cargo run -p codex-app-server-test-client -- \
Use `--capture-file /tmp/plugin-analytics.jsonl` to select the output path.
The command validates one `codex_plugin_disabled`, `codex_plugin_enabled`, and
`codex_plugin_used` event, prints them, and leaves the JSONL file in place for
inspection. It does not install or uninstall plugins and does not modify the
profile's persistent config.
inspection. Each event must retain the local ID in both `plugin_id` and
`local_plugin_id`, and include the backend ID in `remote_plugin_id`. The command
does not install or uninstall plugins and does not modify the profile's
persistent config.
### Testing remote install and uninstall analytics
@@ -58,6 +60,10 @@ installed, installs it, validates `codex_plugin_installed`, uninstalls it,
validates `codex_plugin_uninstalled`, and verifies that the original
uninstalled state was restored.
The mutation events retain the backend ID in the legacy `plugin_id` field,
include the local Codex ID in `local_plugin_id`, and include the backend ID in
`remote_plugin_id`.
`--remote-plugin-id` takes the backend ID, such as `plugins~Plugin_...`, not the
local `<plugin>@<marketplace>` ID.
@@ -5,33 +5,6 @@ use serde_json::Value;
use std::fs;
use std::io;
use std::path::Path;
use std::thread;
use std::time::Duration;
use std::time::Instant;
const CAPTURE_READY_TIMEOUT: Duration = Duration::from_secs(5);
const CAPTURE_READY_POLL_INTERVAL: Duration = Duration::from_millis(50);
pub(super) fn wait_until_capture_is_ready(path: &Path) -> Result<()> {
let deadline = Instant::now() + CAPTURE_READY_TIMEOUT;
loop {
match fs::metadata(path) {
Ok(_) => return Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err)
.with_context(|| format!("inspect capture file {}", path.display()));
}
}
if Instant::now() >= deadline {
anyhow::bail!(
"analytics capture did not become ready at {}; use an authenticated profile and a debug Codex binary",
path.display()
);
}
thread::sleep(CAPTURE_READY_POLL_INTERVAL);
}
}
pub(super) fn read_events_for_remote_plugin(
path: &Path,
@@ -70,7 +43,7 @@ pub(super) fn read_events_for_remote_plugin(
}
pub(super) struct PluginEventIdentity<'a> {
pub(super) plugin_id: &'a str,
pub(super) local_plugin_id: &'a str,
pub(super) remote_plugin_id: &'a str,
pub(super) plugin_name: &'a str,
pub(super) marketplace_name: &'a str,
@@ -101,7 +74,8 @@ pub(super) fn validate_mutation_events(
fn validate_event(event: &Value, expected: &PluginEventIdentity<'_>) -> Result<()> {
let params = &event["event_params"];
require_string(params, "plugin_id", expected.plugin_id)?;
require_string(params, "plugin_id", expected.remote_plugin_id)?;
require_string(params, "local_plugin_id", expected.local_plugin_id)?;
require_string(params, "remote_plugin_id", expected.remote_plugin_id)?;
require_string(params, "plugin_name", expected.plugin_name)?;
require_string(params, "marketplace_name", expected.marketplace_name)?;
@@ -74,7 +74,8 @@ fn mutation_event(event_type: &str) -> Value {
json!({
"event_type": event_type,
"event_params": {
"plugin_id": "sample@openai-curated-remote",
"plugin_id": REMOTE_PLUGIN_ID,
"local_plugin_id": "sample@openai-curated-remote",
"remote_plugin_id": REMOTE_PLUGIN_ID,
"plugin_name": "sample",
"marketplace_name": "openai-curated-remote",
@@ -88,7 +89,7 @@ fn mutation_event(event_type: &str) -> Value {
fn expected_identity() -> PluginEventIdentity<'static> {
PluginEventIdentity {
plugin_id: "sample@openai-curated-remote",
local_plugin_id: "sample@openai-curated-remote",
remote_plugin_id: REMOTE_PLUGIN_ID,
plugin_name: "sample",
marketplace_name: "openai-curated-remote",
@@ -2,9 +2,9 @@ use super::CodexClient;
use super::plugin_analytics_capture::PluginEventIdentity;
use super::plugin_analytics_capture::read_events_for_remote_plugin;
use super::plugin_analytics_capture::validate_mutation_events;
use super::plugin_analytics_capture::wait_until_capture_is_ready;
use super::plugin_analytics_smoke::ANALYTICS_CAPTURE_ENV_VAR;
use super::plugin_analytics_smoke::prepare_capture_file;
use super::plugin_analytics_smoke::wait_until_capture_is_ready;
use anyhow::Context;
use anyhow::Result;
use anyhow::anyhow;
@@ -49,6 +49,7 @@ pub(super) fn run(
prepare_capture_file(&capture_path)?;
let mut client = spawn_client(codex_bin, config_overrides, &capture_path)?;
wait_until_capture_is_ready(&capture_path)?;
client.initialize()?;
let initial = read_remote_plugin(&mut client, remote_plugin_id)?;
validate_initial_plugin(&initial, remote_plugin_id)?;
@@ -127,6 +128,8 @@ pub(super) fn run_cleanup(
});
prepare_capture_file(&capture_path)?;
let mut client = spawn_client(codex_bin, config_overrides, &capture_path)?;
wait_until_capture_is_ready(&capture_path)?;
client.initialize()?;
println!("capture file: {}", capture_path.display());
match restore_uninstalled_state(&mut client, remote_plugin_id) {
@@ -205,9 +208,7 @@ fn spawn_client(
OsString::from(ANALYTICS_CAPTURE_ENV_VAR),
capture_path.as_os_str().to_os_string(),
)];
let mut client = CodexClient::spawn_stdio_with_env(codex_bin, &overrides, &environment)?;
client.initialize()?;
Ok(client)
CodexClient::spawn_stdio_with_env(codex_bin, &overrides, &environment)
}
#[derive(Clone, Debug)]
@@ -319,7 +320,7 @@ fn run_mutation_sequence(
let events = validate_mutation_events(
events,
PluginEventIdentity {
plugin_id: &expected.plugin_id,
local_plugin_id: &expected.plugin_id,
remote_plugin_id: &expected.remote_plugin_id,
plugin_name: &expected.plugin_name,
marketplace_name: &expected.marketplace_name,
@@ -29,6 +29,7 @@ use std::time::Instant;
pub(super) const ANALYTICS_CAPTURE_ENV_VAR: &str = "CODEX_ANALYTICS_EVENTS_CAPTURE_FILE";
const TEST_USER_CONFIG_ENV_VAR: &str = "CODEX_APP_SERVER_TEST_USER_CONFIG_FILE";
const CAPTURE_READY_TIMEOUT: Duration = Duration::from_secs(5);
const CAPTURE_TIMEOUT: Duration = Duration::from_secs(10);
const CAPTURE_POLL_INTERVAL: Duration = Duration::from_millis(50);
const MOCK_MODEL_SLUG: &str = "plugin-analytics-smoke";
@@ -64,6 +65,7 @@ pub(super) fn run(
),
];
let mut client = CodexClient::spawn_stdio_with_env(codex_bin, &overrides, &child_environment)?;
wait_until_capture_is_ready(&capture_path)?;
client.initialize()?;
let installed = plugin_installed(&mut client)?;
@@ -260,6 +262,27 @@ pub(super) fn prepare_capture_file(path: &Path) -> Result<()> {
Ok(())
}
pub(super) fn wait_until_capture_is_ready(path: &Path) -> Result<()> {
let deadline = Instant::now() + CAPTURE_READY_TIMEOUT;
loop {
match fs::metadata(path) {
Ok(_) => return Ok(()),
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(err)
.with_context(|| format!("inspect capture file {}", path.display()));
}
}
if Instant::now() >= deadline {
bail!(
"analytics capture did not become ready at {}; use a debug Codex binary",
path.display()
);
}
thread::sleep(CAPTURE_POLL_INTERVAL);
}
}
fn wait_for_plugin_events(path: &Path, plugin_id: &str) -> Result<Vec<Value>> {
let deadline = Instant::now() + CAPTURE_TIMEOUT;
loop {
@@ -358,6 +381,7 @@ fn event_count(events: &[Value], event_type: &str) -> usize {
fn validate_identity(event: &Value, expected: &ExpectedPlugin) -> Result<()> {
let params = &event["event_params"];
require_string(params, "plugin_id", &expected.plugin_id)?;
require_string(params, "local_plugin_id", &expected.plugin_id)?;
require_string(params, "remote_plugin_id", &expected.remote_plugin_id)?;
require_string(params, "plugin_name", &expected.plugin_name)?;
require_string(params, "marketplace_name", &expected.marketplace_name)
+4
View File
@@ -1156,8 +1156,11 @@ fn analytics_rpc_transport(transport: &AppServerTransport) -> AppServerRpcTransp
#[cfg(test)]
mod tests {
use super::LogFormat;
#[cfg(debug_assertions)]
use super::loader_overrides_with_test_user_config_file;
#[cfg(debug_assertions)]
use codex_config::LoaderOverrides;
#[cfg(debug_assertions)]
use codex_utils_absolute_path::AbsolutePathBuf;
use pretty_assertions::assert_eq;
@@ -1179,6 +1182,7 @@ mod tests {
assert_eq!(LogFormat::from_env_value(Some("jsonl")), LogFormat::Default);
}
#[cfg(debug_assertions)]
#[test]
fn debug_test_user_config_file_overrides_loader_path() {
let path = std::env::temp_dir().join("codex-app-server-test-config.toml");
+31 -8
View File
@@ -62,6 +62,7 @@ use codex_plugin::AppConnectorId;
use codex_plugin::PluginCapabilitySummary;
use codex_plugin::PluginId;
use codex_plugin::PluginIdError;
use codex_plugin::PluginTelemetryLegacyIdSource;
use codex_plugin::PluginTelemetryMetadata;
use codex_plugin::prompt_safe_plugin_description;
use codex_protocol::protocol::HookEventName;
@@ -539,15 +540,34 @@ impl PluginsManager {
}
fn remote_plugin_id_for(&self, plugin_id: &PluginId) -> Option<String> {
let cache = match self.remote_installed_plugins_cache.read() {
Ok(cache) => cache,
Err(err) => err.into_inner(),
let cached_remote_plugin_id = {
let cache = match self.remote_installed_plugins_cache.read() {
Ok(cache) => cache,
Err(err) => err.into_inner(),
};
cache.as_ref().and_then(|plugins| {
plugins.iter().find_map(|plugin| {
(plugin.name == plugin_id.plugin_name
&& plugin.marketplace_name == plugin_id.marketplace_name)
.then(|| plugin.id.clone())
})
})
};
cache.as_ref()?.iter().find_map(|plugin| {
(plugin.name == plugin_id.plugin_name
&& plugin.marketplace_name == plugin_id.marketplace_name)
.then(|| plugin.id.clone())
})
if cached_remote_plugin_id.is_some() {
return cached_remote_plugin_id;
}
match self.store.remote_plugin_id(plugin_id) {
Ok(remote_plugin_id) => remote_plugin_id,
Err(err) => {
tracing::warn!(
plugin_id = %plugin_id.as_key(),
error = %err,
"failed to read persisted remote plugin identity"
);
None
}
}
}
pub async fn telemetry_metadata_for_installed_plugin(
@@ -561,6 +581,7 @@ impl PluginsManager {
PluginTelemetryMetadata {
plugin_id: plugin_id.clone(),
remote_plugin_id: self.remote_plugin_id_for(plugin_id),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary,
}
}
@@ -577,6 +598,7 @@ impl PluginsManager {
PluginTelemetryMetadata {
plugin_id: plugin_id.clone(),
remote_plugin_id: Some(remote_plugin_id.to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Remote,
capability_summary,
}
}
@@ -589,6 +611,7 @@ impl PluginsManager {
Some(PluginTelemetryMetadata {
remote_plugin_id: self.remote_plugin_id_for(&plugin_id),
plugin_id,
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(summary.clone()),
})
}
+75 -3
View File
@@ -384,6 +384,7 @@ async fn installed_plugin_telemetry_metadata_uses_local_identity_without_remote_
PluginTelemetryMetadata {
plugin_id,
remote_plugin_id: None,
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(PluginCapabilitySummary {
config_name: "sample@test".to_string(),
display_name: "sample".to_string(),
@@ -397,13 +398,15 @@ async fn installed_plugin_telemetry_metadata_uses_local_identity_without_remote_
}
#[tokio::test]
async fn installed_plugin_telemetry_metadata_resolves_remote_snapshot_identity() {
async fn installed_plugin_telemetry_metadata_resolves_persisted_remote_identity() {
let codex_home = TempDir::new().unwrap();
write_cached_plugin(codex_home.path(), "openai-curated-remote", "linear");
let manager = PluginsManager::new(codex_home.path().to_path_buf());
manager.write_remote_installed_plugins_cache(vec![remote_installed_linear_plugin()]);
let plugin_id =
PluginId::parse("linear@openai-curated-remote").expect("plugin id should parse");
PluginStore::new(codex_home.path().to_path_buf())
.write_remote_plugin_id(&plugin_id, "plugins~Plugin_linear")
.expect("persist remote plugin id");
let manager = PluginsManager::new(codex_home.path().to_path_buf());
let metadata = manager
.telemetry_metadata_for_installed_plugin(&plugin_id)
@@ -414,6 +417,41 @@ async fn installed_plugin_telemetry_metadata_resolves_remote_snapshot_identity()
PluginTelemetryMetadata {
plugin_id,
remote_plugin_id: Some("plugins~Plugin_linear".to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(PluginCapabilitySummary {
config_name: "linear@openai-curated-remote".to_string(),
display_name: "linear".to_string(),
description: None,
has_skills: true,
mcp_server_names: Vec::new(),
app_connector_ids: Vec::new(),
}),
}
);
}
#[tokio::test]
async fn installed_plugin_telemetry_metadata_resolves_remote_snapshot_identity() {
let codex_home = TempDir::new().unwrap();
write_cached_plugin(codex_home.path(), "openai-curated-remote", "linear");
let plugin_id =
PluginId::parse("linear@openai-curated-remote").expect("plugin id should parse");
PluginStore::new(codex_home.path().to_path_buf())
.write_remote_plugin_id(&plugin_id, "plugins~Plugin_stale")
.expect("persist remote plugin id");
let manager = PluginsManager::new(codex_home.path().to_path_buf());
manager.write_remote_installed_plugins_cache(vec![remote_installed_linear_plugin()]);
let metadata = manager
.telemetry_metadata_for_installed_plugin(&plugin_id)
.await;
assert_eq!(
metadata,
PluginTelemetryMetadata {
plugin_id,
remote_plugin_id: Some("plugins~Plugin_linear".to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(PluginCapabilitySummary {
config_name: "linear@openai-curated-remote".to_string(),
display_name: "linear".to_string(),
@@ -442,6 +480,7 @@ async fn installed_plugin_telemetry_metadata_accepts_authoritative_remote_identi
PluginTelemetryMetadata {
plugin_id,
remote_plugin_id: Some("plugins~Plugin_linear".to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Remote,
capability_summary: None,
}
);
@@ -469,6 +508,39 @@ fn capability_summary_telemetry_metadata_resolves_remote_snapshot_identity() {
plugin_id: PluginId::parse("linear@openai-curated-remote")
.expect("plugin id should parse"),
remote_plugin_id: Some("plugins~Plugin_linear".to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(summary),
})
);
}
#[test]
fn capability_summary_telemetry_metadata_resolves_persisted_remote_identity() {
let codex_home = TempDir::new().unwrap();
write_cached_plugin(codex_home.path(), "openai-curated-remote", "linear");
let plugin_id =
PluginId::parse("linear@openai-curated-remote").expect("plugin id should parse");
PluginStore::new(codex_home.path().to_path_buf())
.write_remote_plugin_id(&plugin_id, "plugins~Plugin_linear")
.expect("persist remote plugin id");
let manager = PluginsManager::new(codex_home.path().to_path_buf());
let summary = PluginCapabilitySummary {
config_name: "linear@openai-curated-remote".to_string(),
display_name: "Linear".to_string(),
description: Some("Track work".to_string()),
has_skills: true,
mcp_server_names: vec!["linear".to_string()],
app_connector_ids: vec![AppConnectorId("linear-app".to_string())],
};
let metadata = manager.telemetry_metadata_for_capability_summary(&summary);
assert_eq!(
metadata,
Some(PluginTelemetryMetadata {
plugin_id,
remote_plugin_id: Some("plugins~Plugin_linear".to_string()),
legacy_plugin_id_source: PluginTelemetryLegacyIdSource::Local,
capability_summary: Some(summary),
})
);
+10
View File
@@ -39,11 +39,21 @@ pub struct PluginHookSource {
pub hooks: HookEventsToml,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PluginTelemetryLegacyIdSource {
/// Preserve the local Codex plugin ID in the legacy `plugin_id` field.
Local,
/// Preserve the backend plugin ID in the legacy `plugin_id` field.
Remote,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PluginTelemetryMetadata {
/// Local plugin identifier used by Codex configuration and the plugin cache.
pub plugin_id: PluginId,
/// Optional backend identifier for remote plugins.
pub remote_plugin_id: Option<String>,
/// Selects the identity serialized into the compatibility `plugin_id` field.
pub legacy_plugin_id_source: PluginTelemetryLegacyIdSource,
pub capability_summary: Option<PluginCapabilitySummary>,
}