mirror of
https://github.com/openai/codex.git
synced 2026-11-19 22:30:42 +00:00
402 lines
13 KiB
Rust
402 lines
13 KiB
Rust
use anyhow::Result;
|
|
use app_test_support::ChatGptAuthFixture;
|
|
use app_test_support::DEFAULT_CLIENT_NAME;
|
|
use app_test_support::McpProcess;
|
|
use app_test_support::write_chatgpt_auth;
|
|
use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url;
|
|
use codex_app_server::in_process;
|
|
use codex_app_server::in_process::InProcessStartArgs;
|
|
use codex_app_server_protocol::ClientInfo;
|
|
use codex_app_server_protocol::InitializeParams;
|
|
use codex_arg0::Arg0DispatchPaths;
|
|
use codex_config::CloudRequirementsLoader;
|
|
use codex_config::LoaderOverrides;
|
|
use codex_config::types::AuthCredentialsStoreMode;
|
|
use codex_config::types::OtelExporterKind;
|
|
use codex_config::types::OtelHttpProtocol;
|
|
use codex_core::config::ConfigBuilder;
|
|
use codex_exec_server::EnvironmentManager;
|
|
use codex_feedback::CodexFeedback;
|
|
use codex_protocol::protocol::SessionSource;
|
|
use pretty_assertions::assert_eq;
|
|
use serde_json::Value;
|
|
use std::collections::HashMap;
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
use tempfile::TempDir;
|
|
use tokio::time::timeout;
|
|
use wiremock::Mock;
|
|
use wiremock::MockServer;
|
|
use wiremock::ResponseTemplate;
|
|
use wiremock::matchers::method;
|
|
use wiremock::matchers::path;
|
|
|
|
const SERVICE_VERSION: &str = "0.0.0-test";
|
|
|
|
fn set_metrics_exporter(config: &mut codex_core::config::Config) {
|
|
config.otel.metrics_exporter = OtelExporterKind::OtlpHttp {
|
|
endpoint: "http://localhost:4318".to_string(),
|
|
headers: HashMap::new(),
|
|
protocol: OtelHttpProtocol::Json,
|
|
tls: None,
|
|
};
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn app_server_default_analytics_disabled_without_flag() -> Result<()> {
|
|
let codex_home = TempDir::new()?;
|
|
let mut config = ConfigBuilder::default()
|
|
.codex_home(codex_home.path().to_path_buf())
|
|
.build()
|
|
.await?;
|
|
set_metrics_exporter(&mut config);
|
|
config.analytics_enabled = None;
|
|
|
|
let provider = codex_core::otel_init::build_provider(
|
|
&config,
|
|
SERVICE_VERSION,
|
|
Some("codex-app-server"),
|
|
/*default_analytics_enabled*/ false,
|
|
)
|
|
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
|
|
|
|
// With analytics unset in the config and the default flag is false, metrics are disabled.
|
|
// A provider may still exist for non-metrics telemetry, so check metrics specifically.
|
|
let has_metrics = provider.as_ref().and_then(|otel| otel.metrics()).is_some();
|
|
assert_eq!(has_metrics, false);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn app_server_default_analytics_enabled_with_flag() -> Result<()> {
|
|
let codex_home = TempDir::new()?;
|
|
let mut config = ConfigBuilder::default()
|
|
.codex_home(codex_home.path().to_path_buf())
|
|
.build()
|
|
.await?;
|
|
set_metrics_exporter(&mut config);
|
|
config.analytics_enabled = None;
|
|
|
|
let provider = codex_core::otel_init::build_provider(
|
|
&config,
|
|
SERVICE_VERSION,
|
|
Some("codex-app-server"),
|
|
/*default_analytics_enabled*/ true,
|
|
)
|
|
.map_err(|err| anyhow::anyhow!(err.to_string()))?;
|
|
|
|
// With analytics unset in the config and the default flag is true, metrics are enabled.
|
|
let has_metrics = provider.as_ref().and_then(|otel| otel.metrics()).is_some();
|
|
assert_eq!(has_metrics, true);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn standalone_app_server_startup_tracks_analytics_event() -> Result<()> {
|
|
let server = MockServer::start().await;
|
|
let codex_home = TempDir::new()?;
|
|
write_mock_responses_config_toml_with_chatgpt_base_url(
|
|
codex_home.path(),
|
|
&server.uri(),
|
|
&server.uri(),
|
|
)?;
|
|
mount_analytics_capture(&server, codex_home.path()).await?;
|
|
|
|
let _mcp = McpProcess::new_without_managed_config(codex_home.path()).await?;
|
|
let event =
|
|
wait_for_analytics_event(&server, Duration::from_secs(10), "codex_app_server_started")
|
|
.await?;
|
|
|
|
assert_app_server_started_event(&event, "stdio");
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn standalone_app_server_startup_respects_default_disabled_analytics() -> Result<()> {
|
|
let server = MockServer::start().await;
|
|
let codex_home = TempDir::new()?;
|
|
write_mock_responses_config_toml_with_chatgpt_base_url(
|
|
codex_home.path(),
|
|
&server.uri(),
|
|
&server.uri(),
|
|
)?;
|
|
mount_analytics_endpoint(&server).await;
|
|
write_analytics_auth(codex_home.path())?;
|
|
|
|
let _mcp = McpProcess::new_without_managed_config(codex_home.path()).await?;
|
|
|
|
assert_analytics_event_not_received(&server, Duration::from_secs(1), "codex_app_server_started")
|
|
.await
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn embedded_app_server_startup_tracks_analytics_event() -> Result<()> {
|
|
let server = MockServer::start().await;
|
|
let codex_home = TempDir::new()?;
|
|
write_mock_responses_config_toml_with_chatgpt_base_url(
|
|
codex_home.path(),
|
|
&server.uri(),
|
|
&server.uri(),
|
|
)?;
|
|
mount_analytics_capture(&server, codex_home.path()).await?;
|
|
|
|
let client =
|
|
in_process::start(in_process_start_args(codex_home.path(), "codex-tui").await?).await?;
|
|
|
|
let event =
|
|
wait_for_analytics_event(&server, Duration::from_secs(10), "codex_app_server_started")
|
|
.await?;
|
|
|
|
assert_app_server_started_event(&event, "in_process");
|
|
client.shutdown().await?;
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn embedded_app_server_failed_initialize_does_not_track_startup_event() -> Result<()> {
|
|
let server = MockServer::start().await;
|
|
let codex_home = TempDir::new()?;
|
|
write_mock_responses_config_toml_with_chatgpt_base_url(
|
|
codex_home.path(),
|
|
&server.uri(),
|
|
&server.uri(),
|
|
)?;
|
|
mount_analytics_capture(&server, codex_home.path()).await?;
|
|
|
|
let result =
|
|
in_process::start(in_process_start_args(codex_home.path(), "bad\rname").await?).await;
|
|
let Err(error) = result else {
|
|
anyhow::bail!("in-process start should reject an invalid client name");
|
|
};
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("in-process initialize failed: Invalid clientInfo.name")
|
|
);
|
|
|
|
assert_analytics_event_not_received(&server, Duration::from_secs(1), "codex_app_server_started")
|
|
.await
|
|
}
|
|
|
|
async fn in_process_start_args(codex_home: &Path, client_name: &str) -> Result<InProcessStartArgs> {
|
|
let loader_overrides = LoaderOverrides::without_managed_config_for_tests();
|
|
let config = ConfigBuilder::default()
|
|
.codex_home(codex_home.to_path_buf())
|
|
.fallback_cwd(Some(codex_home.to_path_buf()))
|
|
.loader_overrides(loader_overrides.clone())
|
|
.build()
|
|
.await?;
|
|
Ok(InProcessStartArgs {
|
|
arg0_paths: Arg0DispatchPaths::default(),
|
|
config: Arc::new(config),
|
|
cli_overrides: Vec::new(),
|
|
loader_overrides,
|
|
strict_config: false,
|
|
cloud_requirements: CloudRequirementsLoader::default(),
|
|
thread_config_loader: Arc::new(codex_config::NoopThreadConfigLoader),
|
|
feedback: CodexFeedback::new(),
|
|
log_db: None,
|
|
state_db: None,
|
|
environment_manager: Arc::new(EnvironmentManager::default_for_tests()),
|
|
config_warnings: Vec::new(),
|
|
session_source: SessionSource::Cli,
|
|
enable_codex_api_key_env: false,
|
|
initialize: InitializeParams {
|
|
client_info: ClientInfo {
|
|
name: client_name.to_string(),
|
|
title: None,
|
|
version: "0.1.0".to_string(),
|
|
},
|
|
capabilities: None,
|
|
},
|
|
channel_capacity: in_process::DEFAULT_IN_PROCESS_CHANNEL_CAPACITY,
|
|
})
|
|
}
|
|
|
|
fn assert_app_server_started_event(event: &Value, rpc_transport: &str) {
|
|
assert_eq!(event["event_params"]["rpc_transport"], rpc_transport);
|
|
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
|
|
assert!(event["event_params"]["created_at"].as_u64().is_some());
|
|
assert!(
|
|
event["event_params"]["runtime"]["codex_rs_version"]
|
|
.as_str()
|
|
.is_some()
|
|
);
|
|
}
|
|
|
|
pub(crate) async fn mount_analytics_capture(server: &MockServer, codex_home: &Path) -> Result<()> {
|
|
mount_analytics_endpoint(server).await;
|
|
|
|
let config_path = codex_home.join("config.toml");
|
|
let config = std::fs::read_to_string(&config_path)?;
|
|
std::fs::write(
|
|
config_path,
|
|
format!("{config}\n[analytics]\nenabled = true\n"),
|
|
)?;
|
|
|
|
write_analytics_auth(codex_home)
|
|
}
|
|
|
|
async fn mount_analytics_endpoint(server: &MockServer) {
|
|
Mock::given(method("POST"))
|
|
.and(path("/codex/analytics-events/events"))
|
|
.respond_with(ResponseTemplate::new(200))
|
|
.mount(server)
|
|
.await;
|
|
}
|
|
|
|
fn write_analytics_auth(codex_home: &Path) -> Result<()> {
|
|
write_chatgpt_auth(
|
|
codex_home,
|
|
ChatGptAuthFixture::new("chatgpt-token")
|
|
.account_id("account-123")
|
|
.chatgpt_user_id("user-123")
|
|
.chatgpt_account_id("account-123"),
|
|
AuthCredentialsStoreMode::File,
|
|
)?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
async fn assert_analytics_event_not_received(
|
|
server: &MockServer,
|
|
wait_duration: Duration,
|
|
event_type: &str,
|
|
) -> Result<()> {
|
|
tokio::time::sleep(wait_duration).await;
|
|
let Some(requests) = server.received_requests().await else {
|
|
return Ok(());
|
|
};
|
|
for request in &requests {
|
|
if request.method != "POST" || request.url.path() != "/codex/analytics-events/events" {
|
|
continue;
|
|
}
|
|
let payload: Value = serde_json::from_slice(&request.body)
|
|
.map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?;
|
|
if payload["events"]
|
|
.as_array()
|
|
.is_some_and(|events| events.iter().any(|event| event["event_type"] == event_type))
|
|
{
|
|
anyhow::bail!("received unexpected {event_type} analytics event");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) async fn wait_for_thread_initialized_payload(
|
|
server: &MockServer,
|
|
read_timeout: Duration,
|
|
) -> Result<Value> {
|
|
timeout(read_timeout, async {
|
|
loop {
|
|
let Some(requests) = server.received_requests().await else {
|
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
|
continue;
|
|
};
|
|
for request in &requests {
|
|
if request.method != "POST"
|
|
|| request.url.path() != "/codex/analytics-events/events"
|
|
{
|
|
continue;
|
|
}
|
|
let payload: Value = serde_json::from_slice(&request.body)
|
|
.map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?;
|
|
if thread_initialized_event(&payload).is_ok() {
|
|
return Ok::<Value, anyhow::Error>(payload);
|
|
}
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
|
}
|
|
})
|
|
.await?
|
|
}
|
|
|
|
pub(crate) async fn wait_for_analytics_event(
|
|
server: &MockServer,
|
|
read_timeout: Duration,
|
|
event_type: &str,
|
|
) -> Result<Value> {
|
|
timeout(read_timeout, async {
|
|
loop {
|
|
let Some(requests) = server.received_requests().await else {
|
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
|
continue;
|
|
};
|
|
for request in &requests {
|
|
if request.method != "POST"
|
|
|| request.url.path() != "/codex/analytics-events/events"
|
|
{
|
|
continue;
|
|
}
|
|
let payload: Value = serde_json::from_slice(&request.body)
|
|
.map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?;
|
|
let Some(events) = payload["events"].as_array() else {
|
|
continue;
|
|
};
|
|
if let Some(event) = events
|
|
.iter()
|
|
.find(|event| event["event_type"] == event_type)
|
|
{
|
|
return Ok::<Value, anyhow::Error>(event.clone());
|
|
}
|
|
}
|
|
tokio::time::sleep(Duration::from_millis(25)).await;
|
|
}
|
|
})
|
|
.await?
|
|
}
|
|
|
|
pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> {
|
|
let events = payload["events"]
|
|
.as_array()
|
|
.ok_or_else(|| anyhow::anyhow!("analytics payload missing events array"))?;
|
|
events
|
|
.iter()
|
|
.find(|event| event["event_type"] == "codex_thread_initialized")
|
|
.ok_or_else(|| anyhow::anyhow!("codex_thread_initialized event should be present"))
|
|
}
|
|
|
|
pub(crate) fn assert_basic_thread_initialized_event(
|
|
event: &Value,
|
|
thread_id: &str,
|
|
session_id: &str,
|
|
expected_model: &str,
|
|
initialization_mode: &str,
|
|
expected_thread_source: &str,
|
|
) {
|
|
assert_eq!(event["event_params"]["thread_id"], thread_id);
|
|
assert_eq!(event["event_params"]["session_id"], session_id);
|
|
assert_eq!(
|
|
event["event_params"]["app_server_client"]["product_client_id"],
|
|
DEFAULT_CLIENT_NAME
|
|
);
|
|
assert_eq!(
|
|
event["event_params"]["app_server_client"]["client_name"],
|
|
DEFAULT_CLIENT_NAME
|
|
);
|
|
assert_eq!(
|
|
event["event_params"]["app_server_client"]["rpc_transport"],
|
|
"stdio"
|
|
);
|
|
assert_eq!(event["event_params"]["model"], expected_model);
|
|
assert_eq!(event["event_params"]["ephemeral"], false);
|
|
assert_eq!(
|
|
event["event_params"]["thread_source"],
|
|
expected_thread_source
|
|
);
|
|
assert_eq!(
|
|
event["event_params"]["subagent_source"],
|
|
serde_json::Value::Null
|
|
);
|
|
assert_eq!(
|
|
event["event_params"]["parent_thread_id"],
|
|
serde_json::Value::Null
|
|
);
|
|
assert_eq!(
|
|
event["event_params"]["initialization_mode"],
|
|
initialization_mode
|
|
);
|
|
assert!(event["event_params"]["created_at"].as_u64().is_some());
|
|
}
|