mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Add aggregate metrics for status line provider usage (#62041)
## What changed - Count status line picker openings and successful provider changes with fixed labels for enabling a custom provider, switching custom providers, and returning to the default. Skip unchanged selections and failed saves. - Apply current analytics consent to the built-in exporter. Suppress built-in reporting for custom environments, unrecognized originators, or custom OpenTelemetry resource attributes and headers, while allowing explicitly configured OTLP collectors. - Stop recording through the captured collector if the configured metrics exporter changes. ## Testing Add assertions for picker and provider-change counters, unchanged selections, failed saves, custom resource and header suppression, and exporter changes. GitOrigin-RevId: 600bc06d6587e83a6ca09241f0285b47d30a331e
This commit is contained in:
@@ -561,6 +561,7 @@ struct InitialHistoryReplayBuffer {
|
||||
|
||||
pub(crate) struct App {
|
||||
statusline_picker: Option<statusline_picker::StatuslinePicker>,
|
||||
statusline_metrics: crate::statusline_metrics::StatuslineMetrics,
|
||||
provider_writes: HashMap<crate::backend_id::BackendId, crate::model_catalog::ProviderWrite>,
|
||||
feature_write_lock: Arc<tokio::sync::Mutex<()>>,
|
||||
config_notification_test_generation: u64,
|
||||
|
||||
@@ -249,6 +249,7 @@ impl App {
|
||||
tui: &mut tui::Tui,
|
||||
initial_backends: super::backend_sessions::InitialBackendSessions,
|
||||
mut config: Config,
|
||||
statusline_metrics: crate::statusline_metrics::StatuslineMetrics,
|
||||
launch_cwd: PathBuf,
|
||||
backend_preference_context: backend_selection::BackendPreferenceContext,
|
||||
cli_kv_overrides: Vec<(String, TomlValue)>,
|
||||
@@ -988,6 +989,7 @@ See the Codex keymap documentation for supported actions and examples."
|
||||
agents_overview::AgentsOverviewState::new(local_settings.tui.agents_overview_grouping);
|
||||
let mut app = Self {
|
||||
statusline_picker: None,
|
||||
statusline_metrics,
|
||||
provider_writes,
|
||||
feature_write_lock: Arc::default(),
|
||||
config_notification_test_generation: 0,
|
||||
|
||||
@@ -7,6 +7,8 @@ use crate::statusline_provider::StatuslineProvider;
|
||||
use anyhow::Context;
|
||||
use codex_config::types::BuiltinStatusLine;
|
||||
use codex_config::types::StatusLineSelection;
|
||||
use codex_config::types::StatusLineSelection::Builtin;
|
||||
use codex_config::types::StatusLineSelection::Mcp;
|
||||
|
||||
impl App {
|
||||
pub(super) async fn save_status_line_provider(
|
||||
@@ -15,8 +17,8 @@ impl App {
|
||||
provider_from_picker: Option<StatuslineProvider>,
|
||||
) -> anyhow::Result<()> {
|
||||
let value = match &provider {
|
||||
StatusLineSelection::Builtin(BuiltinStatusLine::Default) => toml_edit::value("default"),
|
||||
StatusLineSelection::Mcp(id) => {
|
||||
Builtin(BuiltinStatusLine::Default) => toml_edit::value("default"),
|
||||
Mcp(id) => {
|
||||
let table = toml_edit::InlineTable::from_iter([
|
||||
("server", id.server.as_str()),
|
||||
("tool", id.tool.as_str()),
|
||||
@@ -33,6 +35,25 @@ impl App {
|
||||
.await
|
||||
.context("Failed to save status line provider")?;
|
||||
|
||||
let previous = self
|
||||
.local_settings
|
||||
.tui
|
||||
.status_line_provider
|
||||
.as_deref()
|
||||
.unwrap_or(&Builtin(BuiltinStatusLine::Default));
|
||||
let change = match (previous, &provider) {
|
||||
(Builtin(_), Mcp(_)) => Some("enabled_custom"),
|
||||
(Mcp(_), Builtin(_)) => Some("returned_to_default"),
|
||||
(Mcp(old), Mcp(new)) if old != new => Some("switched_custom"),
|
||||
(Builtin(_), Builtin(_)) | (Mcp(_), Mcp(_)) => None,
|
||||
};
|
||||
if let Some(change) = change {
|
||||
self.statusline_metrics.record(
|
||||
&self.config,
|
||||
"codex.statusline.provider_changed",
|
||||
&[("change", change)],
|
||||
);
|
||||
}
|
||||
self.local_settings.tui.status_line_provider = Some(Box::new(provider.clone()));
|
||||
self.chat_widget.local_settings.tui.status_line_provider = Some(Box::new(provider));
|
||||
self.chat_widget
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use crate::AppServerTarget;
|
||||
use crate::app::test_support::make_test_app;
|
||||
use crate::chatwidget::tests::render_bottom_popup;
|
||||
use crate::statusline_metrics::tests::Collector;
|
||||
use codex_config::types::BuiltinStatusLine;
|
||||
use codex_config::types::StatusLineProviderId;
|
||||
use codex_config::types::StatusLineSelection;
|
||||
@@ -101,13 +102,17 @@ async fn remote_session_saves_to_the_active_local_profile() -> anyhow::Result<()
|
||||
#[tokio::test]
|
||||
async fn failed_write_preserves_the_file_and_the_live_selection() -> anyhow::Result<()> {
|
||||
let mut app = make_test_app().await;
|
||||
let collector = Collector::new();
|
||||
app.statusline_metrics = collector.metrics(&mut app.config);
|
||||
let path = app.local_settings.user_config_path.clone();
|
||||
let active = StatusLineSelection::Mcp(StatusLineProviderId {
|
||||
server: "work".into(),
|
||||
tool: "usage".into(),
|
||||
});
|
||||
app.save_status_line_provider(active, /*provider_from_picker*/ None)
|
||||
.await?;
|
||||
for tool in ["usage", "other", "other"] {
|
||||
let selection = StatusLineSelection::Mcp(StatusLineProviderId {
|
||||
server: "work".into(),
|
||||
tool: tool.into(),
|
||||
});
|
||||
app.save_status_line_provider(selection, /*provider_from_picker*/ None)
|
||||
.await?;
|
||||
}
|
||||
let before = (
|
||||
app.local_settings.clone(),
|
||||
app.chat_widget.local_settings.clone(),
|
||||
@@ -126,6 +131,13 @@ async fn failed_write_preserves_the_file_and_the_live_selection() -> anyhow::Res
|
||||
(&before.0, &before.1)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
collector.snapshot(),
|
||||
[
|
||||
"provider_changed change=enabled_custom 1",
|
||||
"provider_changed change=switched_custom 1"
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -185,6 +185,8 @@ impl App {
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.statusline_metrics
|
||||
.record(&self.config, "codex.statusline.picker_opened", &[]);
|
||||
let selected = self
|
||||
.chat_widget
|
||||
.configured_statusline_provider()
|
||||
|
||||
@@ -6,6 +6,7 @@ use crate::app::tests::make_test_app_with_channels;
|
||||
use crate::app::tests::test_thread_session;
|
||||
use crate::app_server_session::ThreadParamsMode;
|
||||
use crate::chatwidget::tests::render_bottom_popup;
|
||||
use crate::statusline_metrics::tests::Collector;
|
||||
use crate::statusline_provider::test_support::advertised;
|
||||
use crate::tui::test_support::make_test_tui;
|
||||
use codex_app_server_client::AppServerClient;
|
||||
@@ -149,9 +150,13 @@ async fn statusline_picker_disconnect_leaves_the_item_editor_open() -> anyhow::R
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn discovery_batch_keeps_the_highlight() -> anyhow::Result<()> {
|
||||
let (mut app, owner, _commands, _events) = picker().await?;
|
||||
let collector = Collector::new();
|
||||
app.statusline_metrics = collector.metrics(&mut app.config);
|
||||
let calendar = advertised("work", "calendar", "ui://work/day");
|
||||
app.chat_widget.local_settings.tui.status_line_provider =
|
||||
app.local_settings.tui.status_line_provider =
|
||||
Some(Box::new(StatusLineSelection::Mcp((&calendar.id).into())));
|
||||
app.chat_widget.local_settings.tui.status_line_provider =
|
||||
app.local_settings.tui.status_line_provider.clone();
|
||||
let update = |server: &str, providers| DiscoveryUpdate::Server(server.into(), Ok(providers));
|
||||
app.open_statusline_picker(&owner);
|
||||
let scan = app.statusline_picker.as_ref().unwrap().id;
|
||||
@@ -184,8 +189,10 @@ async fn discovery_batch_keeps_the_highlight() -> anyhow::Result<()> {
|
||||
&owner,
|
||||
);
|
||||
insta::assert_snapshot!(render_bottom_popup(&app.chat_widget, /*width*/ 67));
|
||||
assert_eq!(collector.snapshot(), ["picker_opened 1"]);
|
||||
|
||||
app.chat_widget.handle_key_event(KeyCode::Esc.into());
|
||||
app.chat_widget.pause_unavailable_thread();
|
||||
app.reconnect.offline = true;
|
||||
app.open_statusline_picker(&owner);
|
||||
insta::assert_snapshot!(
|
||||
@@ -204,6 +211,13 @@ async fn discovery_batch_keeps_the_highlight() -> anyhow::Result<()> {
|
||||
app.local_settings.tui.status_line_provider.as_deref(),
|
||||
Some(&StatusLineSelection::Builtin(BuiltinStatusLine::Default))
|
||||
);
|
||||
assert_eq!(
|
||||
collector.snapshot(),
|
||||
[
|
||||
"picker_opened 1",
|
||||
"provider_changed change=returned_to_default 1"
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ pub(crate) async fn make_test_app() -> Box<App> {
|
||||
|
||||
Box::new(App {
|
||||
statusline_picker: None,
|
||||
statusline_metrics: Default::default(),
|
||||
provider_writes: HashMap::new(),
|
||||
feature_write_lock: Arc::default(),
|
||||
config_notification_test_generation: 0,
|
||||
|
||||
@@ -9177,6 +9177,7 @@ async fn make_test_app() -> Box<App> {
|
||||
|
||||
Box::new(App {
|
||||
statusline_picker: None,
|
||||
statusline_metrics: Default::default(),
|
||||
provider_writes: HashMap::new(),
|
||||
feature_write_lock: Arc::default(),
|
||||
config_notification_test_generation: 0,
|
||||
@@ -9322,6 +9323,7 @@ pub(super) async fn make_test_app_with_channels() -> (
|
||||
(
|
||||
Box::new(App {
|
||||
statusline_picker: None,
|
||||
statusline_metrics: Default::default(),
|
||||
provider_writes: HashMap::new(),
|
||||
feature_write_lock: Arc::default(),
|
||||
config_notification_test_generation: 0,
|
||||
|
||||
@@ -28,6 +28,7 @@ pub(super) async fn run_startup_for_test(
|
||||
tui,
|
||||
initial_backends,
|
||||
config.clone(),
|
||||
Default::default(),
|
||||
config.cwd.to_path_buf(),
|
||||
backend_selection::BackendPreferenceContext::new(
|
||||
&config,
|
||||
|
||||
@@ -232,6 +232,7 @@ mod startup_presentation;
|
||||
mod startup_recovery;
|
||||
mod status;
|
||||
mod status_indicator_widget;
|
||||
mod statusline_metrics;
|
||||
mod statusline_provider;
|
||||
mod streaming;
|
||||
mod style;
|
||||
@@ -1161,6 +1162,7 @@ async fn run_ratatui_app(
|
||||
mut app_server_target: AppServerTarget,
|
||||
remote_cwd_override: Option<PathBuf>,
|
||||
initial_config: Config,
|
||||
statusline_metrics: statusline_metrics::StatuslineMetrics,
|
||||
manually_selected_oss_provider: Option<String>,
|
||||
overrides: ConfigOverrides,
|
||||
cli_kv_overrides: Vec<(String, toml::Value)>,
|
||||
@@ -2295,6 +2297,7 @@ async fn run_ratatui_app(
|
||||
&mut tui,
|
||||
initial_backends,
|
||||
config,
|
||||
statusline_metrics,
|
||||
current_cwd.to_path_buf(),
|
||||
backend_preference_context,
|
||||
cli_kv_overrides.clone(),
|
||||
|
||||
@@ -729,6 +729,11 @@ pub(super) async fn run_main_inner(
|
||||
.as_ref()
|
||||
.and_then(codex_otel::OtelProvider::metrics)
|
||||
.cloned();
|
||||
let statusline_metrics = crate::statusline_metrics::StatuslineMetrics::new(
|
||||
&config,
|
||||
metrics.clone(),
|
||||
&otel_originator,
|
||||
);
|
||||
if let Some(metrics) = &metrics {
|
||||
let _ = codex_otel::record_process_start_once(metrics, otel_originator.as_str());
|
||||
let telemetry =
|
||||
@@ -1009,6 +1014,7 @@ pub(super) async fn run_main_inner(
|
||||
app_server_target,
|
||||
remote_cwd_override,
|
||||
config,
|
||||
statusline_metrics,
|
||||
manually_selected_oss_provider,
|
||||
overrides,
|
||||
cli_kv_overrides,
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
//! Aggregate statusline counters with fixed labels and the current analytics consent.
|
||||
|
||||
use crate::legacy_core::config::Config;
|
||||
use codex_config::types::DEFAULT_OTEL_ENVIRONMENT;
|
||||
use codex_config::types::OtelExporterKind;
|
||||
use codex_otel::MetricsClient;
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct StatuslineMetrics {
|
||||
collector: Option<(MetricsClient, OtelExporterKind)>,
|
||||
}
|
||||
|
||||
impl StatuslineMetrics {
|
||||
pub(crate) fn new(config: &Config, metrics: Option<MetricsClient>, originator: &str) -> Self {
|
||||
Self::with_env(config, metrics, originator, |key| std::env::var_os(key))
|
||||
}
|
||||
|
||||
fn with_env(
|
||||
config: &Config,
|
||||
metrics: Option<MetricsClient>,
|
||||
originator: &str,
|
||||
get: impl Fn(&str) -> Option<std::ffi::OsString>,
|
||||
) -> Self {
|
||||
let exporter = &config.otel.metrics_exporter;
|
||||
// The exporter captures resource attributes and headers at launch. Custom values may
|
||||
// identify the installation and belong only in an explicitly configured collector.
|
||||
let eligible = match exporter {
|
||||
OtelExporterKind::None => false,
|
||||
OtelExporterKind::Statsig => {
|
||||
config.otel.environment == DEFAULT_OTEL_ENVIRONMENT
|
||||
&& codex_otel::bounded_originator_tag_value(originator) == originator
|
||||
&& [
|
||||
"OTEL_RESOURCE_ATTRIBUTES",
|
||||
"OTEL_EXPORTER_OTLP_METRICS_HEADERS",
|
||||
"OTEL_EXPORTER_OTLP_HEADERS",
|
||||
]
|
||||
.into_iter()
|
||||
.all(|key| get(key).is_none_or(|value| value.is_empty()))
|
||||
}
|
||||
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. } => true,
|
||||
};
|
||||
Self {
|
||||
collector: metrics
|
||||
.filter(|_| eligible)
|
||||
.map(|client| (client, exporter.clone())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record(
|
||||
&self,
|
||||
config: &Config,
|
||||
name: &'static str,
|
||||
labels: &[(&'static str, &'static str)],
|
||||
) {
|
||||
if let Some((client, exporter)) = &self.collector
|
||||
&& exporter == &config.otel.metrics_exporter
|
||||
{
|
||||
let _ = client.counter_with_analytics_consent(
|
||||
name,
|
||||
/*inc*/ 1,
|
||||
labels,
|
||||
config.analytics_enabled != Some(false),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "statusline_metrics_tests.rs"]
|
||||
pub(crate) mod tests;
|
||||
@@ -0,0 +1,101 @@
|
||||
#![cfg(test)]
|
||||
//! Collect the full label set so identifying attributes fail the same assertions as wrong counts.
|
||||
|
||||
use super::*;
|
||||
use codex_otel::MetricsConfig;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use opentelemetry_sdk::metrics::data::ScopeMetrics;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
pub(crate) struct Collector {
|
||||
client: MetricsClient,
|
||||
}
|
||||
|
||||
impl Collector {
|
||||
pub(crate) fn new() -> Self {
|
||||
let config = MetricsConfig::in_memory(
|
||||
"test",
|
||||
"codex-tui",
|
||||
"test",
|
||||
InMemoryMetricExporter::default(),
|
||||
)
|
||||
.with_runtime_reader();
|
||||
Self {
|
||||
client: MetricsClient::new(config).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn metrics(&self, config: &mut Config) -> StatuslineMetrics {
|
||||
config.otel.metrics_exporter = OtelExporterKind::OtlpGrpc {
|
||||
endpoint: "http://localhost:4317".into(),
|
||||
headers: Default::default(),
|
||||
tls: None,
|
||||
};
|
||||
StatuslineMetrics::new(config, Some(self.client.clone()), "codex-tui")
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> Vec<String> {
|
||||
let batch = self.client.snapshot().unwrap();
|
||||
let mut result = Vec::new();
|
||||
for metric in batch.scope_metrics().flat_map(ScopeMetrics::metrics) {
|
||||
let Some(name) = metric.name().strip_prefix("codex.statusline.") else {
|
||||
continue;
|
||||
};
|
||||
let AggregatedMetrics::U64(MetricData::Sum(sum)) = metric.data() else {
|
||||
panic!("expected a counter for {name}");
|
||||
};
|
||||
for point in sum.data_points() {
|
||||
let labels: String = point
|
||||
.attributes()
|
||||
.map(|a| format!(" {}={}", a.key, a.value))
|
||||
.collect();
|
||||
result.push(format!("{name}{labels} {}", point.value()));
|
||||
}
|
||||
}
|
||||
result.sort_unstable();
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn custom_resources_and_otlp_headers_suppress_builtin_reporting() {
|
||||
let mut app = crate::app::test_support::make_test_app().await;
|
||||
app.config.otel.metrics_exporter = OtelExporterKind::Statsig;
|
||||
let collector = Collector::new();
|
||||
for key in [
|
||||
None,
|
||||
Some("OTEL_RESOURCE_ATTRIBUTES"),
|
||||
Some("OTEL_EXPORTER_OTLP_METRICS_HEADERS"),
|
||||
Some("OTEL_EXPORTER_OTLP_HEADERS"),
|
||||
] {
|
||||
let metrics = StatuslineMetrics::with_env(
|
||||
&app.config,
|
||||
Some(collector.client.clone()),
|
||||
"codex-tui",
|
||||
|name| (key == Some(name)).then(|| "private=value".into()),
|
||||
);
|
||||
metrics.record(&app.config, "codex.statusline.picker_opened", &[]);
|
||||
let expected = if key.is_none() {
|
||||
vec!["picker_opened 1"]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
assert_eq!(collector.snapshot(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metrics_skip_changed_destinations_and_custom_builtin_resources() {
|
||||
let mut app = crate::app::test_support::make_test_app().await;
|
||||
let collector = Collector::new();
|
||||
let metrics = collector.metrics(&mut app.config);
|
||||
metrics.record(&app.config, "codex.statusline.picker_opened", &[]);
|
||||
app.config.otel.metrics_exporter = OtelExporterKind::Statsig;
|
||||
metrics.record(&app.config, "codex.statusline.picker_opened", &[]);
|
||||
app.config.otel.environment = "custom".into();
|
||||
let metrics = StatuslineMetrics::new(&app.config, Some(collector.client.clone()), "codex-tui");
|
||||
metrics.record(&app.config, "codex.statusline.picker_opened", &[]);
|
||||
assert_eq!(collector.snapshot(), ["picker_opened 1"]);
|
||||
}
|
||||
Reference in New Issue
Block a user