Export turn cost as an OTEL metric (#40488)

## What changed

- Emit `codex.turn.cost_microusd` as a counter with turn, conversation,
  interruption, speed, and reasoning-effort attributes. Convert the estimated
  USD string to microdollars, rounding to the nearest microdollar and skipping
  values that cannot be represented safely.
- Start turn-cost collection when an OTLP metrics exporter is configured, even
  when the OTEL log exporter is disabled.

## Testing

- Verify cost conversion, rounding, and metric attributes with an in-memory
  metrics snapshot.
- Verify that a metrics-only OTLP configuration starts the turn-cost worker.

GitOrigin-RevId: e60587079f097f10cc1b6d1bf889d3f7a0dc3d5f
This commit is contained in:
xli-oai
2026-08-24 22:44:33 +00:00
committed by copyberry
parent fb9311db5c
commit 2497518490
7 changed files with 151 additions and 2 deletions
+8 -2
View File
@@ -95,10 +95,16 @@ enum BackendAvailability {
impl TurnCostWorker {
pub(crate) fn spawn(config: Arc<Config>, auth_manager: Arc<AuthManager>) -> Option<Self> {
if !matches!(
let has_otel_log_exporter = matches!(
config.otel.exporter,
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
) || config.model_provider.is_amazon_bedrock()
);
let has_otel_metrics_exporter = matches!(
config.otel.metrics_exporter,
OtelExporterKind::OtlpHttp { .. } | OtelExporterKind::OtlpGrpc { .. }
);
if !(has_otel_log_exporter || has_otel_metrics_exporter)
|| config.model_provider.is_amazon_bedrock()
{
return None;
}
@@ -21,6 +21,39 @@ use wiremock::matchers::path;
const TURN_COST_PATH: &str = "/v1/analytics/codex/turn-costs";
#[tokio::test]
async fn worker_starts_with_otlp_metrics_exporter_without_log_exporter() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path(TURN_COST_PATH))
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
"turns": []
})))
.expect(1)
.mount(&server)
.await;
let codex_home = TempDir::new().expect("temporary Codex home");
let mut config = ConfigBuilder::default()
.codex_home(codex_home.path().to_path_buf())
.build()
.await
.expect("test config");
config.chatgpt_base_url = server.uri();
config.otel.exporter = OtelExporterKind::None;
config.otel.metrics_exporter = OtelExporterKind::OtlpGrpc {
endpoint: server.uri(),
headers: HashMap::new(),
tls: None,
};
let auth_manager = AuthManager::from_auth_for_testing(CodexAuth::from_api_key("sk-test"));
let worker = TurnCostWorker::spawn(Arc::new(config), auth_manager)
.expect("OTLP metrics exporter should enable turn-cost collection");
wait_for_request_count(&server, /*expected*/ 1).await;
worker.shutdown();
server.verify().await;
}
#[tokio::test]
async fn handle_observes_only_matching_model_provider() {
let codex_home = TempDir::new().expect("temporary Codex home");
@@ -23,6 +23,7 @@ use crate::metrics::STARTUP_PHASE_DURATION_METRIC;
use crate::metrics::SessionMetricTagValues;
use crate::metrics::TOOL_CALL_COUNT_METRIC;
use crate::metrics::TOOL_CALL_DURATION_METRIC;
use crate::metrics::TURN_COST_MICROUSD_METRIC;
use crate::metrics::TURN_TTFT_DURATION_METRIC;
use crate::metrics::WEBSOCKET_EVENT_COUNT_METRIC;
use crate::metrics::WEBSOCKET_EVENT_DURATION_METRIC;
@@ -281,6 +282,45 @@ impl SessionTelemetry {
speed: Option<&str>,
reasoning_effort: Option<&str>,
) {
let (dollars, fractional) = estimated_usd.split_once('.').unwrap_or((estimated_usd, ""));
let fractional = fractional.as_bytes();
let fractional_precision = 6_usize;
let estimated_microusd = dollars.parse::<u64>().ok().and_then(|dollars| {
if !fractional.iter().all(u8::is_ascii_digit) {
return None;
}
let fractional_microusd = fractional
.iter()
.take(fractional_precision)
.fold(0_u64, |value, digit| value * 10 + u64::from(digit - b'0'))
* 10_u64.pow(fractional_precision.saturating_sub(fractional.len()) as u32);
let round_up = fractional
.get(fractional_precision)
.is_some_and(|digit| *digit >= b'5');
let estimated_microusd = dollars
.checked_mul(1_000_000)?
.checked_add(fractional_microusd)?
.checked_add(u64::from(round_up))?;
i64::try_from(estimated_microusd).ok()
});
if let Some(estimated_microusd) = estimated_microusd {
let conversation_id = self.metadata.conversation_id.to_string();
let mut tags = vec![
("turn.id", turn_id),
("conversation.id", conversation_id.as_str()),
(
"turn.interrupted",
if interrupted { "true" } else { "false" },
),
];
if let Some(speed) = speed {
tags.push(("speed", speed));
}
if let Some(reasoning_effort) = reasoning_effort {
tags.push(("reasoning_effort", reasoning_effort));
}
self.counter(TURN_COST_MICROUSD_METRIC, estimated_microusd, &tags);
}
log_event!(
self,
event.name = "codex.turn_cost",
+2
View File
@@ -7,6 +7,7 @@ use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TBT_DURATION_METRIC;
use crate::metrics::names::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
use crate::metrics::names::TOOL_CALL_COUNT_METRIC;
use crate::metrics::names::TOOL_CALL_DURATION_METRIC;
use crate::metrics::names::TURN_COST_MICROUSD_METRIC;
use crate::metrics::names::TURN_TOKEN_USAGE_METRIC;
use crate::metrics::validation::validate_tag_key;
use crate::metrics::validation::validate_tag_value;
@@ -27,6 +28,7 @@ const STATSIG_DISABLED_METRICS: &[&str] = &[
RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC,
TOOL_CALL_COUNT_METRIC,
TOOL_CALL_DURATION_METRIC,
TURN_COST_MICROUSD_METRIC,
TURN_TOKEN_USAGE_METRIC,
];
+1
View File
@@ -31,6 +31,7 @@ pub const TURN_NETWORK_PROXY_METRIC: &str = "codex.turn.network_proxy";
pub const TURN_MEMORY_METRIC: &str = "codex.turn.memory";
pub const TURN_TOOL_CALL_METRIC: &str = "codex.turn.tool.call";
pub const TURN_TOKEN_USAGE_METRIC: &str = "codex.turn.token_usage";
pub const TURN_COST_MICROUSD_METRIC: &str = "codex.turn.cost_microusd";
pub const TURN_UNIFIED_EXEC_RUNNING_PROCESSES_METRIC: &str =
"codex.turn.unified_exec.running_processes";
pub const GUARDIAN_REVIEW_COUNT_METRIC: &str = "codex.guardian.review";
+2
View File
@@ -604,6 +604,7 @@ mod tests {
use crate::metrics::RESPONSES_API_ENGINE_SERVICE_TTFT_DURATION_METRIC;
use crate::metrics::TOOL_CALL_COUNT_METRIC;
use crate::metrics::TOOL_CALL_DURATION_METRIC;
use crate::metrics::TURN_COST_MICROUSD_METRIC;
use crate::metrics::TURN_TOKEN_USAGE_METRIC;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use pretty_assertions::assert_eq;
@@ -741,6 +742,7 @@ mod tests {
)?;
metrics.counter(TOOL_CALL_COUNT_METRIC, /*inc*/ 1, &[])?;
metrics.record_duration(TOOL_CALL_DURATION_METRIC, Duration::from_millis(25), &[])?;
metrics.counter(TURN_COST_MICROUSD_METRIC, /*inc*/ 1, &[])?;
metrics.histogram(TURN_TOKEN_USAGE_METRIC, /*value*/ 100, &[])?;
metrics.counter("codex.turns", /*inc*/ 1, &[])?;
metrics.shutdown()?;
+65
View File
@@ -155,3 +155,68 @@ fn manager_snapshot_metrics_collects_without_shutdown() -> Result<()> {
Ok(())
}
#[test]
fn manager_turn_cost_records_microusd_metric() -> Result<()> {
let exporter = InMemoryMetricExporter::default();
let config = MetricsConfig::in_memory("test", "codex-cli", env!("CARGO_PKG_VERSION"), exporter)
.with_runtime_reader();
let metrics = MetricsClient::new(config)?;
let thread_id = ThreadId::new();
let conversation_id = thread_id.to_string();
let manager = SessionTelemetry::new(
thread_id,
"gpt-5.6",
"gpt-5.6",
/*account_id*/ None,
/*account_email*/ None,
Some(TelemetryAuthMode::ApiKey),
"test_originator".to_string(),
/*log_user_prompts*/ false,
"tty".to_string(),
SessionSource::Cli,
)
.with_metrics(metrics);
manager.record_turn_cost(
"turn-123",
"0.0001245",
/*interrupted*/ false,
Some("fast"),
Some("high"),
);
let snapshot = manager.snapshot_metrics()?;
let metric = find_metric(&snapshot, "codex.turn.cost_microusd")
.expect("turn-cost microdollar metric missing");
let point = match metric.data() {
AggregatedMetrics::U64(MetricData::Sum(sum)) => {
sum.data_points().next().expect("turn-cost data point")
}
_ => panic!("unexpected turn-cost metric data type"),
};
assert_eq!(point.value(), 125);
assert_eq!(
attributes_to_map(point.attributes()),
BTreeMap::from([
(
"app.version".to_string(),
env!("CARGO_PKG_VERSION").to_string(),
),
(
"auth_mode".to_string(),
TelemetryAuthMode::ApiKey.to_string(),
),
("conversation.id".to_string(), conversation_id),
("model".to_string(), "gpt-5.6".to_string()),
("originator".to_string(), "test_originator".to_string()),
("reasoning_effort".to_string(), "high".to_string()),
("session_source".to_string(), "cli".to_string()),
("speed".to_string(), "fast".to_string()),
("turn.id".to_string(), "turn-123".to_string()),
("turn.interrupted".to_string(), "false".to_string()),
])
);
Ok(())
}