token usage

This commit is contained in:
Roy Han
2026-04-01 15:38:23 -07:00
parent 916866937f
commit 6821427024
5 changed files with 215 additions and 278 deletions
@@ -46,6 +46,9 @@ use codex_app_server_protocol::Thread;
use codex_app_server_protocol::ThreadResumeResponse;
use codex_app_server_protocol::ThreadStartResponse;
use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus;
use codex_app_server_protocol::ThreadTokenUsage;
use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification;
use codex_app_server_protocol::TokenUsageBreakdown;
use codex_app_server_protocol::Turn;
use codex_app_server_protocol::TurnCompletedNotification;
use codex_app_server_protocol::TurnError as AppServerTurnError;
@@ -216,12 +219,40 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact {
}
}
fn sample_thread_token_usage_updated_notification(
thread_id: &str,
turn_id: &str,
) -> ServerNotification {
ServerNotification::ThreadTokenUsageUpdated(ThreadTokenUsageUpdatedNotification {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_string(),
token_usage: ThreadTokenUsage {
total: TokenUsageBreakdown {
total_tokens: 500,
input_tokens: 200,
cached_input_tokens: 50,
output_tokens: 220,
reasoning_output_tokens: 30,
},
last: TokenUsageBreakdown {
total_tokens: 321,
input_tokens: 123,
cached_input_tokens: 45,
output_tokens: 140,
reasoning_output_tokens: 13,
},
model_context_window: Some(200_000),
},
})
}
async fn ingest_turn_prerequisites(
reducer: &mut AnalyticsReducer,
out: &mut Vec<TrackEventRequest>,
include_initialize: bool,
include_resolved_config: bool,
include_started: bool,
include_token_usage: bool,
) {
if include_initialize {
reducer
@@ -291,6 +322,17 @@ async fn ingest_turn_prerequisites(
)
.await;
}
if include_token_usage {
reducer
.ingest(
AnalyticsFact::Notification(Box::new(
sample_thread_token_usage_updated_notification("thread-2", "turn-2"),
)),
out,
)
.await;
}
}
fn expected_absolute_path(path: &PathBuf) -> String {
@@ -956,6 +998,7 @@ async fn turn_lifecycle_emits_turn_event() {
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ true,
/*include_token_usage*/ true,
)
.await;
reducer
@@ -984,6 +1027,14 @@ async fn turn_lifecycle_emits_turn_event() {
assert_eq!(payload["event_params"]["created_at"], json!(455));
assert_eq!(payload["event_params"]["completed_at"], json!(456));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
assert_eq!(payload["event_params"]["input_tokens"], json!(123));
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45));
assert_eq!(payload["event_params"]["output_tokens"], json!(140));
assert_eq!(
payload["event_params"]["reasoning_output_tokens"],
json!(13)
);
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
}
#[tokio::test]
@@ -997,6 +1048,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
/*include_initialize*/ false,
/*include_resolved_config*/ true,
/*include_started*/ false,
/*include_token_usage*/ false,
)
.await;
reducer
@@ -1026,6 +1078,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
/*include_initialize*/ true,
/*include_resolved_config*/ false,
/*include_started*/ false,
/*include_token_usage*/ false,
)
.await;
reducer
@@ -1053,6 +1106,7 @@ async fn turn_completed_without_started_notification_emits_null_created_at() {
/*include_initialize*/ true,
/*include_resolved_config*/ true,
/*include_started*/ false,
/*include_token_usage*/ false,
)
.await;
reducer
@@ -1070,6 +1124,14 @@ async fn turn_completed_without_started_notification_emits_null_created_at() {
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
assert_eq!(payload["event_params"]["created_at"], json!(null));
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
assert_eq!(payload["event_params"]["input_tokens"], json!(null));
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null));
assert_eq!(payload["event_params"]["output_tokens"], json!(null));
assert_eq!(
payload["event_params"]["reasoning_output_tokens"],
json!(null)
);
assert_eq!(payload["event_params"]["total_tokens"], json!(null));
}
fn sample_plugin_metadata() -> PluginTelemetryMetadata {
+46 -26
View File
@@ -34,6 +34,7 @@ use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo;
use codex_app_server_protocol::InitializeParams;
use codex_app_server_protocol::RequestId;
use codex_app_server_protocol::ServerNotification;
use codex_app_server_protocol::TokenUsageBreakdown;
use codex_app_server_protocol::UserInput;
use codex_git_utils::collect_git_info;
use codex_git_utils::get_git_repo_root;
@@ -83,6 +84,7 @@ struct TurnState {
thread_id: Option<String>,
num_input_images: Option<usize>,
resolved_config: Option<TurnResolvedConfigFact>,
token_usage: Option<TokenUsageBreakdown>,
created_at: Option<u64>,
completed: Option<CompletedTurnState>,
}
@@ -206,6 +208,7 @@ impl AnalyticsReducer {
thread_id: None,
num_input_images: None,
resolved_config: None,
token_usage: None,
created_at: None,
completed: None,
});
@@ -358,6 +361,7 @@ impl AnalyticsReducer {
thread_id: None,
num_input_images: None,
resolved_config: None,
token_usage: None,
created_at: None,
completed: None,
});
@@ -382,11 +386,27 @@ impl AnalyticsReducer {
thread_id: None,
num_input_images: None,
resolved_config: None,
token_usage: None,
created_at: None,
completed: None,
});
turn_state.created_at = u64::try_from(notification.created_at).ok();
}
ServerNotification::ThreadTokenUsageUpdated(notification) => {
let turn_state =
self.turns
.entry(notification.turn_id.clone())
.or_insert(TurnState {
connection_id: None,
thread_id: None,
num_input_images: None,
resolved_config: None,
token_usage: None,
created_at: None,
completed: None,
});
turn_state.token_usage = Some(notification.token_usage.last);
}
ServerNotification::TurnCompleted(notification) => {
let turn_state =
self.turns
@@ -396,6 +416,7 @@ impl AnalyticsReducer {
thread_id: None,
num_input_images: None,
resolved_config: None,
token_usage: None,
created_at: None,
completed: None,
});
@@ -453,18 +474,13 @@ impl AnalyticsReducer {
let Some(turn_state) = self.turns.get(turn_id) else {
return;
};
let Some(thread_id) = turn_state.thread_id.clone() else {
if turn_state.thread_id.is_none()
|| turn_state.num_input_images.is_none()
|| turn_state.resolved_config.is_none()
|| turn_state.completed.is_none()
{
return;
};
let Some(num_input_images) = turn_state.num_input_images else {
return;
};
let Some(resolved_config) = turn_state.resolved_config.clone() else {
return;
};
let Some(completed) = turn_state.completed.clone() else {
return;
};
}
let product_client_id = turn_state
.connection_id
.and_then(|connection_id| self.connections.get(&connection_id))
@@ -475,12 +491,8 @@ impl AnalyticsReducer {
event_type: "codex_turn_event",
event_params: codex_turn_event_params(
product_client_id,
thread_id,
turn_id.to_string(),
num_input_images,
resolved_config,
completed,
turn_state.created_at,
turn_state,
),
},
)));
@@ -490,13 +502,19 @@ impl AnalyticsReducer {
fn codex_turn_event_params(
product_client_id: String,
thread_id: String,
turn_id: String,
num_input_images: usize,
resolved_config: TurnResolvedConfigFact,
completed: CompletedTurnState,
created_at: Option<u64>,
turn_state: &TurnState,
) -> CodexTurnEventParams {
let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = (
turn_state.thread_id.clone(),
turn_state.num_input_images,
turn_state.resolved_config.clone(),
turn_state.completed.clone(),
) else {
unreachable!("turn event params require a fully populated turn state");
};
let token_usage = turn_state.token_usage.clone();
let created_at = turn_state.created_at;
let TurnResolvedConfigFact {
turn_id: _resolved_turn_id,
thread_id: _resolved_thread_id,
@@ -546,11 +564,13 @@ fn codex_turn_event_params(
subagent_tool_call_count: None,
web_search_count: None,
image_generation_count: None,
input_tokens: None,
cached_input_tokens: None,
output_tokens: None,
reasoning_output_tokens: None,
total_tokens: None,
input_tokens: token_usage.as_ref().map(|usage| usage.input_tokens),
cached_input_tokens: token_usage.as_ref().map(|usage| usage.cached_input_tokens),
output_tokens: token_usage.as_ref().map(|usage| usage.output_tokens),
reasoning_output_tokens: token_usage
.as_ref()
.map(|usage| usage.reasoning_output_tokens),
total_tokens: token_usage.as_ref().map(|usage| usage.total_tokens),
duration_ms: completed.duration_ms,
created_at,
completed_at: Some(completed.completed_at),
@@ -309,6 +309,84 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
assert!(event["event_params"]["created_at"].as_u64().is_some());
assert!(event["event_params"]["completed_at"].as_u64().is_some());
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
assert_eq!(event["event_params"]["input_tokens"], 0);
assert_eq!(event["event_params"]["cached_input_tokens"], 0);
assert_eq!(event["event_params"]["output_tokens"], 0);
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
assert_eq!(event["event_params"]["total_tokens"], 0);
Ok(())
}
#[tokio::test]
async fn turn_start_tracks_turn_event_analytics_for_empty_input() -> Result<()> {
let responses = vec![create_final_assistant_message_sse_response("Done")?];
let server = create_mock_responses_server_sequence_unchecked(responses).await;
let codex_home = TempDir::new()?;
write_mock_responses_config_toml_with_chatgpt_base_url(
codex_home.path(),
&server.uri(),
&server.uri(),
)?;
enable_analytics_capture(&server, codex_home.path()).await?;
let mut mcp = McpProcess::new(codex_home.path()).await?;
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
let thread_req = mcp
.send_thread_start_request(ThreadStartParams {
model: Some("mock-model".to_string()),
..Default::default()
})
.await?;
let thread_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
)
.await??;
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
let turn_req = mcp
.send_turn_start_request(TurnStartParams {
thread_id: thread.id.clone(),
input: Vec::new(),
..Default::default()
})
.await?;
let turn_resp: JSONRPCResponse = timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
)
.await??;
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
timeout(
DEFAULT_READ_TIMEOUT,
mcp.read_stream_until_notification_message("turn/completed"),
)
.await??;
let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?;
assert_eq!(event["event_params"]["thread_id"], thread.id);
assert_eq!(event["event_params"]["turn_id"], turn.id);
assert_eq!(
event["event_params"]["product_client_id"],
DEFAULT_CLIENT_NAME
);
assert_eq!(event["event_params"]["model"], "mock-model");
assert_eq!(event["event_params"]["model_provider"], "mock_provider");
assert_eq!(event["event_params"]["sandbox_policy"], "read_only");
assert_eq!(event["event_params"]["num_input_images"], 0);
assert_eq!(event["event_params"]["status"], "completed");
assert!(event["event_params"]["created_at"].as_u64().is_some());
assert!(event["event_params"]["completed_at"].as_u64().is_some());
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
assert_eq!(event["event_params"]["input_tokens"], 0);
assert_eq!(event["event_params"]["cached_input_tokens"], 0);
assert_eq!(event["event_params"]["output_tokens"], 0);
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
assert_eq!(event["event_params"]["total_tokens"], 0);
Ok(())
}
+29 -31
View File
@@ -5877,37 +5877,35 @@ pub(crate) async fn run_turn(
.await;
}
if !input.is_empty() {
let is_first_turn = {
let mut state = sess.state.lock().await;
state.take_next_turn_is_first()
};
sess.services
.analytics_events_client
.track_turn_resolved_config(TurnResolvedConfigFact {
turn_id: tracking.turn_id.clone(),
thread_id: tracking.thread_id.clone(),
num_input_images: input
.iter()
.filter(|item| {
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
})
.count(),
submission_type: None,
model: turn_context.model_info.slug.clone(),
model_provider: turn_context.config.model_provider_id.clone(),
sandbox_policy: turn_context.sandbox_policy.get().clone(),
reasoning_effort: turn_context.reasoning_effort,
reasoning_summary: Some(turn_context.reasoning_summary),
service_tier: turn_context.config.service_tier,
approval_policy: turn_context.approval_policy.value(),
approvals_reviewer: turn_context.config.approvals_reviewer,
sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(),
collaboration_mode: turn_context.collaboration_mode.mode,
personality: turn_context.personality,
is_first_turn,
});
}
let is_first_turn = {
let mut state = sess.state.lock().await;
state.take_next_turn_is_first()
};
sess.services
.analytics_events_client
.track_turn_resolved_config(TurnResolvedConfigFact {
turn_id: tracking.turn_id.clone(),
thread_id: tracking.thread_id.clone(),
num_input_images: input
.iter()
.filter(|item| {
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
})
.count(),
submission_type: None,
model: turn_context.model_info.slug.clone(),
model_provider: turn_context.config.model_provider_id.clone(),
sandbox_policy: turn_context.sandbox_policy.get().clone(),
reasoning_effort: turn_context.reasoning_effort,
reasoning_summary: Some(turn_context.reasoning_summary),
service_tier: turn_context.config.service_tier,
approval_policy: turn_context.approval_policy.value(),
approvals_reviewer: turn_context.config.approvals_reviewer,
sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(),
collaboration_mode: turn_context.collaboration_mode.mode,
personality: turn_context.personality,
is_first_turn,
});
let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref());
sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token())
-221
View File
@@ -1,23 +1,16 @@
#![cfg(not(target_os = "windows"))]
use anyhow::Ok;
use codex_core::CodexAuth;
use codex_protocol::config_types::CollaborationMode;
use codex_protocol::config_types::ModeKind;
use codex_protocol::config_types::Personality;
use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::config_types::ServiceTier;
use codex_protocol::config_types::Settings;
use codex_protocol::items::AgentMessageContent;
use codex_protocol::items::TurnItem;
use codex_protocol::models::WebSearchAction;
use codex_protocol::openai_models::ReasoningEffort;
use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::EventMsg;
use codex_protocol::protocol::ItemCompletedEvent;
use codex_protocol::protocol::ItemStartedEvent;
use codex_protocol::protocol::Op;
use codex_protocol::protocol::SandboxPolicy;
use codex_protocol::user_input::ByteRange;
use codex_protocol::user_input::TextElement;
use codex_protocol::user_input::UserInput;
@@ -42,11 +35,8 @@ use core_test_support::test_codex::test_codex;
use core_test_support::wait_for_event;
use core_test_support::wait_for_event_match;
use pretty_assertions::assert_eq;
use serde_json::Value;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;
use std::time::Instant;
fn image_generation_artifact_path(codex_home: &Path, session_id: &str, call_id: &str) -> PathBuf {
fn sanitize(value: &str) -> String {
@@ -130,217 +120,6 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn user_turn_tracks_turn_metadata_analytics() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
mount_sse_once(
&server,
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
)
.await;
let chatgpt_base_url = server.uri();
let mut builder = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
config.chatgpt_base_url = chatgpt_base_url;
config.personality = Some(Personality::None);
});
let TestCodex {
codex,
session_configured,
config,
..
} = builder.build(&server).await?;
codex
.submit(Op::UserTurn {
items: vec![UserInput::Text {
text: "hello turn metadata analytics".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
cwd: config.cwd.to_path_buf(),
approval_policy: AskForApproval::Never,
approvals_reviewer: None,
sandbox_policy: SandboxPolicy::new_read_only_policy(),
model: session_configured.model.clone(),
effort: Some(ReasoningEffort::High),
summary: Some(ReasoningSummary::Detailed),
service_tier: Some(Some(ServiceTier::Flex)),
collaboration_mode: Some(CollaborationMode {
mode: ModeKind::Plan,
settings: Settings {
model: session_configured.model.clone(),
reasoning_effort: Some(ReasoningEffort::High),
developer_instructions: None,
},
}),
personality: None,
})
.await?;
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
let deadline = Instant::now() + Duration::from_secs(10);
let event = loop {
let requests = server.received_requests().await.unwrap_or_default();
let turn_events = requests
.into_iter()
.filter(|request| request.url.path() == "/codex/analytics-events/events")
.filter_map(|request| serde_json::from_slice::<Value>(&request.body).ok())
.flat_map(|payload| {
payload["events"]
.as_array()
.cloned()
.unwrap_or_default()
.into_iter()
})
.filter(|event| event["event_type"] == "codex_turn_event")
.collect::<Vec<_>>();
if let Some(event) = turn_events.last().cloned() {
break event;
}
if Instant::now() >= deadline {
panic!("timed out waiting for turn analytics event");
}
tokio::time::sleep(Duration::from_millis(50)).await;
};
let event_params = &event["event_params"];
assert_eq!(event_params["sandbox_policy"], "read_only");
assert_eq!(
event_params["product_client_id"],
serde_json::json!(codex_core::default_client::originator().value)
);
assert_eq!(event_params["model"], session_configured.model);
assert_eq!(event_params["model_provider"], config.model_provider_id);
assert_eq!(event_params["reasoning_effort"], "high");
assert_eq!(event_params["reasoning_summary"], "detailed");
assert_eq!(event_params["service_tier"], "flex");
assert_eq!(event_params["approval_policy"], "never");
assert_eq!(
event_params["approvals_reviewer"],
config.approvals_reviewer.to_string()
);
assert_eq!(
event_params["sandbox_network_access"],
config.permissions.network_sandbox_policy.is_enabled()
);
assert_eq!(event_params["collaboration_mode"], "plan");
assert!(event_params["personality"].is_null());
assert_eq!(event_params["num_input_images"], 0);
assert_eq!(event_params["is_first_turn"], true);
assert!(event_params["thread_id"].as_str().is_some());
assert!(event_params["turn_id"].as_str().is_some());
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn resumed_thread_turn_tracks_is_not_first_turn_analytics() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));
let server = start_mock_server().await;
mount_sse_once(
&server,
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
)
.await;
mount_sse_once(
&server,
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
)
.await;
let chatgpt_base_url = server.uri();
let mut builder = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
config.chatgpt_base_url = chatgpt_base_url;
});
let initial = builder.build(&server).await?;
initial
.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "first turn".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
})
.await?;
wait_for_event(&initial.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let rollout_path = initial
.session_configured
.rollout_path
.clone()
.expect("persisted rollout path should be present");
let home = initial.home.clone();
let resumed_chatgpt_base_url = server.uri();
let mut resume_builder = test_codex()
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
.with_config(move |config| {
config.chatgpt_base_url = resumed_chatgpt_base_url;
});
let resumed = resume_builder.resume(&server, home, rollout_path).await?;
resumed
.codex
.submit(Op::UserInput {
items: vec![UserInput::Text {
text: "resumed turn".into(),
text_elements: Vec::new(),
}],
final_output_json_schema: None,
})
.await?;
wait_for_event(&resumed.codex, |event| {
matches!(event, EventMsg::TurnComplete(_))
})
.await;
let deadline = Instant::now() + Duration::from_secs(10);
let resumed_turn_event = loop {
let requests = server.received_requests().await.unwrap_or_default();
let turn_events = requests
.iter()
.filter(|request| request.url.path() == "/codex/analytics-events/events")
.filter_map(|request| serde_json::from_slice::<Value>(&request.body).ok())
.flat_map(|payload| {
payload["events"]
.as_array()
.cloned()
.unwrap_or_default()
.into_iter()
})
.filter(|event| event["event_type"] == "codex_turn_event")
.collect::<Vec<_>>();
if let Some(event) = turn_events.last().cloned()
&& turn_events.len() >= 2
{
break event;
}
if Instant::now() >= deadline {
panic!("timed out waiting for resumed turn analytics event");
}
tokio::time::sleep(Duration::from_millis(50)).await;
};
assert_eq!(resumed_turn_event["event_params"]["is_first_turn"], false);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn assistant_message_item_is_emitted() -> anyhow::Result<()> {
skip_if_no_network!(Ok(()));