mirror of
https://github.com/openai/codex.git
synced 2026-11-05 20:10:33 +00:00
Export turn cost as an OTEL metric (#40488)
## What changed - Emit `codex.turn.cost_microusd` as a counter with turn, conversation, interruption, speed, and reasoning-effort attributes. Convert the estimated USD string to microdollars, rounding to the nearest microdollar and skipping values that cannot be represented safely. - Start turn-cost collection when an OTLP metrics exporter is configured, even when the OTEL log exporter is disabled. ## Testing - Verify cost conversion, rounding, and metric attributes with an in-memory metrics snapshot. - Verify that a metrics-only OTLP configuration starts the turn-cost worker. GitOrigin-RevId: e60587079f097f10cc1b6d1bf889d3f7a0dc3d5f
This commit is contained in:
@@ -95,10 +95,16 @@ enum BackendAvailability {
|
||||
|
||||
impl TurnCostWorker {
|
||||
pub(crate) fn spawn(config: Arc<Config>, auth_manager: Arc<AuthManager>) -> Option<Self> {
|
||||
if !matches!(
|
||||
let has_otel_log_exporter = matches!(
|
||||
config.otel.exporter,
|
||||
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
|
||||
) || config.model_provider.is_amazon_bedrock()
|
||||
);
|
||||
let has_otel_metrics_exporter = matches!(
|
||||
config.otel.metrics_exporter,
|
||||
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
|
||||
);
|
||||
if !(has_otel_log_exporter || has_otel_metrics_exporter)
|
||||
|| config.model_provider.is_amazon_bedrock()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,39 @@ use wiremock::matchers::path;
|
||||
|
||||
const TURN_COST_PATH: &str = "/v1/analytics/codex/turn-costs";
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_starts_with_otlp_metrics_exporter_without_log_exporter() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path(TURN_COST_PATH))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||
"turns": []
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let codex_home = TempDir::new().expect("temporary Codex home");
|
||||
let mut config = ConfigBuilder::default()
|
||||
.codex_home(codex_home.path().to_path_buf())
|
||||
.build()
|
||||
.await
|
||||
.expect("test config");
|
||||
config.chatgpt_base_url = server.uri();
|
||||
config.otel.exporter = OtelExporterKind::None;
|
||||
config.otel.metrics_exporter = OtelExporterKind::OtlpGrpc {
|
||||
endpoint: server.uri(),
|
||||
headers: HashMap::new(),
|
||||
tls: None,
|
||||
};
|
||||
let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("sk-test"));
|
||||
|
||||
let worker = TurnCostWorker::spawn(Arc::new(config), auth_manager)
|
||||
.expect("OTLP metrics exporter should enable turn-cost collection");
|
||||
wait_for_request_count(&server, /*expected*/ 1).await;
|
||||
worker.shutdown();
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_observes_only_matching_model_provider() {
|
||||
let codex_home = TempDir::new().expect("temporary Codex home");
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::metrics::STARTUP_PHASE_DURATION_METRIC;
|
||||
use crate::metrics::SessionMetricTagValues;
|
||||
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
||||
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
||||
use crate::metrics::TURN_COST_MICROUSD_METRIC;
|
||||
use crate::metrics::TURN_TTFT_DURATION_METRIC;
|
||||
use crate::metrics::WEBSOCKET_EVENT_COUNT_METRIC;
|
||||
use crate::metrics::WEBSOCKET_EVENT_DURATION_METRIC;
|
||||
@@ -281,6 +282,45 @@ impl SessionTelemetry {
|
||||
speed: Option<&str>,
|
||||
reasoning_effort: Option<&str>,
|
||||
) {
|
||||
let (dollars, fractional) = estimated_usd.split_once('.').unwrap_or((estimated_usd, ""));
|
||||
let fractional = fractional.as_bytes();
|
||||
let fractional_precision = 6_usize;
|
||||
let estimated_microusd = dollars.parse::<u64>().ok().and_then(|dollars| {
|
||||
if !fractional.iter().all(u8::is_ascii_digit) {
|
||||
return None;
|
||||
}
|
||||
let fractional_microusd = fractional
|
||||
.iter()
|
||||
.take(fractional_precision)
|
||||
.fold(0_u64, |value, digit| value * 10 + u64::from(digit - b'0'))
|
||||
* 10_u64.pow(fractional_precision.saturating_sub(fractional.len()) as u32);
|
||||
let round_up = fractional
|
||||
.get(fractional_precision)
|
||||
.is_some_and(|digit| *digit >= b'5');
|
||||
let estimated_microusd = dollars
|
||||
.checked_mul(1_000_000)?
|
||||
.checked_add(fractional_microusd)?
|
||||
.checked_add(u64::from(round_up))?;
|
||||
i64::try_from(estimated_microusd).ok()
|
||||
});
|
||||
if let Some(estimated_microusd) = estimated_microusd {
|
||||
let conversation_id = self.metadata.conversation_id.to_string();
|
||||
let mut tags = vec![
|
||||
("turn.id", turn_id),
|
||||
("conversation.id", conversation_id.as_str()),
|
||||
(
|
||||
"turn.interrupted",
|
||||
if interrupted { "true" } else { "false" },
|
||||
),
|
||||
];
|
||||
if let Some(speed) = speed {
|
||||
tags.push(("speed", speed));
|
||||
}
|
||||
if let Some(reasoning_effort) = reasoning_effort {
|
||||
tags.push(("reasoning_effort", reasoning_effort));
|
||||
}
|
||||
self.counter(TURN_COST_MICROUSD_METRIC, estimated_microusd, &tags);
|
||||
}
|
||||
log_event!(
|
||||
self,
|
||||
event.name = "codex.turn_cost",
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TBT_DURATION_METRIC;
|
||||
use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
||||
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
|
||||
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
|
||||
use crate::metrics::names::TURN_COST_MICROUSD_METRIC;
|
||||
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
|
||||
use crate::metrics::validation::validate_tag_key;
|
||||
use crate::metrics::validation::validate_tag_value;
|
||||
@@ -27,6 +28,7 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
|
||||
RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC,
|
||||
TOOL_CALL_COUNT_METRIC,
|
||||
TOOL_CALL_DURATION_METRIC,
|
||||
TURN_COST_MICROUSD_METRIC,
|
||||
TURN_TOKEN_USAGE_METRIC,
|
||||
];
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ pub const TURN_NETWORK_PROXY_METRIC: &str = "codex.turn.network_proxy";
|
||||
pub const TURN_MEMORY_METRIC: &str = "codex.turn.memory";
|
||||
pub const TURN_TOOL_CALL_METRIC: &str = "codex.turn.tool.call";
|
||||
pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage";
|
||||
pub const TURN_COST_MICROUSD_METRIC: &str = "codex.turn.cost_microusd";
|
||||
pub const TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC: &str =
|
||||
"codex.turn.unified_exec.running_processes";
|
||||
pub const GUARDIAN_REVIEW_COUNT_METRIC: &str = "codex.guardian.review";
|
||||
|
||||
@@ -604,6 +604,7 @@ mod tests {
|
||||
use crate::metrics::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
|
||||
use crate::metrics::TOOL_CALL_COUNT_METRIC;
|
||||
use crate::metrics::TOOL_CALL_DURATION_METRIC;
|
||||
use crate::metrics::TURN_COST_MICROUSD_METRIC;
|
||||
use crate::metrics::TURN_TOKEN_USAGE_METRIC;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -741,6 +742,7 @@ mod tests {
|
||||
)?;
|
||||
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
|
||||
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
|
||||
metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?;
|
||||
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
|
||||
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
|
||||
metrics.shutdown()?;
|
||||
|
||||
@@ -155,3 +155,68 @@ fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manager_turn_cost_records_microusd_metric() -> Result<()> {
|
||||
let exporter = InMemoryMetricExporter::default();
|
||||
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
|
||||
.with_runtime_reader();
|
||||
let metrics = MetricsClient::new(config)?;
|
||||
let thread_id = ThreadId::new();
|
||||
let conversation_id = thread_id.to_string();
|
||||
let manager = SessionTelemetry::new(
|
||||
thread_id,
|
||||
"gpt-5.6",
|
||||
"gpt-5.6",
|
||||
/*account_id*/ None,
|
||||
/*account_email*/ None,
|
||||
Some(TelemetryAuthMode::ApiKey),
|
||||
"test_originator".to_string(),
|
||||
/*log_user_prompts*/ false,
|
||||
"tty".to_string(),
|
||||
SessionSource::Cli,
|
||||
)
|
||||
.with_metrics(metrics);
|
||||
|
||||
manager.record_turn_cost(
|
||||
"turn-123",
|
||||
"0.0001245",
|
||||
/*interrupted*/ false,
|
||||
Some("fast"),
|
||||
Some("high"),
|
||||
);
|
||||
|
||||
let snapshot = manager.snapshot_metrics()?;
|
||||
let metric = find_metric(&snapshot, "codex.turn.cost_microusd")
|
||||
.expect("turn-cost microdollar metric missing");
|
||||
let point = match metric.data() {
|
||||
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
|
||||
sum.data_points().next().expect("turn-cost data point")
|
||||
}
|
||||
_ => panic!("unexpected turn-cost metric data type"),
|
||||
};
|
||||
assert_eq!(point.value(), 125);
|
||||
assert_eq!(
|
||||
attributes_to_map(point.attributes()),
|
||||
BTreeMap::from([
|
||||
(
|
||||
"app.version".to_string(),
|
||||
env!("CARGO_PKG_VERSION").to_string(),
|
||||
),
|
||||
(
|
||||
"auth_mode".to_string(),
|
||||
TelemetryAuthMode::ApiKey.to_string(),
|
||||
),
|
||||
("conversation.id".to_string(), conversation_id),
|
||||
("model".to_string(), "gpt-5.6".to_string()),
|
||||
("originator".to_string(), "test_originator".to_string()),
|
||||
("reasoning_effort".to_string(), "high".to_string()),
|
||||
("session_source".to_string(), "cli".to_string()),
|
||||
("speed".to_string(), "fast".to_string()),
|
||||
("turn.id".to_string(), "turn-123".to_string()),
|
||||
("turn.interrupted".to_string(), "false".to_string()),
|
||||
])
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user