mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
token usage
This commit is contained in:
@@ -46,6 +46,9 @@ use codex_app_server_protocol::Thread;
|
||||
use codex_app_server_protocol::ThreadResumeResponse;
|
||||
use codex_app_server_protocol::ThreadStartResponse;
|
||||
use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus;
|
||||
use codex_app_server_protocol::ThreadTokenUsage;
|
||||
use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification;
|
||||
use codex_app_server_protocol::TokenUsageBreakdown;
|
||||
use codex_app_server_protocol::Turn;
|
||||
use codex_app_server_protocol::TurnCompletedNotification;
|
||||
use codex_app_server_protocol::TurnError as AppServerTurnError;
|
||||
@@ -216,12 +219,40 @@ fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact {
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_thread_token_usage_updated_notification(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
) -> ServerNotification {
|
||||
ServerNotification::ThreadTokenUsageUpdated(ThreadTokenUsageUpdatedNotification {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: turn_id.to_string(),
|
||||
token_usage: ThreadTokenUsage {
|
||||
total: TokenUsageBreakdown {
|
||||
total_tokens: 500,
|
||||
input_tokens: 200,
|
||||
cached_input_tokens: 50,
|
||||
output_tokens: 220,
|
||||
reasoning_output_tokens: 30,
|
||||
},
|
||||
last: TokenUsageBreakdown {
|
||||
total_tokens: 321,
|
||||
input_tokens: 123,
|
||||
cached_input_tokens: 45,
|
||||
output_tokens: 140,
|
||||
reasoning_output_tokens: 13,
|
||||
},
|
||||
model_context_window: Some(200_000),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
async fn ingest_turn_prerequisites(
|
||||
reducer: &mut AnalyticsReducer,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
include_initialize: bool,
|
||||
include_resolved_config: bool,
|
||||
include_started: bool,
|
||||
include_token_usage: bool,
|
||||
) {
|
||||
if include_initialize {
|
||||
reducer
|
||||
@@ -291,6 +322,17 @@ async fn ingest_turn_prerequisites(
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
if include_token_usage {
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Notification(Box::new(
|
||||
sample_thread_token_usage_updated_notification("thread-2", "turn-2"),
|
||||
)),
|
||||
out,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn expected_absolute_path(path: &PathBuf) -> String {
|
||||
@@ -956,6 +998,7 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ true,
|
||||
/*include_token_usage*/ true,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -984,6 +1027,14 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
assert_eq!(payload["event_params"]["created_at"], json!(455));
|
||||
assert_eq!(payload["event_params"]["completed_at"], json!(456));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
assert_eq!(payload["event_params"]["input_tokens"], json!(123));
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45));
|
||||
assert_eq!(payload["event_params"]["output_tokens"], json!(140));
|
||||
assert_eq!(
|
||||
payload["event_params"]["reasoning_output_tokens"],
|
||||
json!(13)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -997,6 +1048,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
|
||||
/*include_initialize*/ false,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1026,6 +1078,7 @@ async fn turn_does_not_emit_without_required_prerequisites() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ false,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1053,6 +1106,7 @@ async fn turn_completed_without_started_notification_emits_null_created_at() {
|
||||
/*include_initialize*/ true,
|
||||
/*include_resolved_config*/ true,
|
||||
/*include_started*/ false,
|
||||
/*include_token_usage*/ false,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
@@ -1070,6 +1124,14 @@ async fn turn_completed_without_started_notification_emits_null_created_at() {
|
||||
let payload = serde_json::to_value(&out[0]).expect("serialize turn event");
|
||||
assert_eq!(payload["event_params"]["created_at"], json!(null));
|
||||
assert_eq!(payload["event_params"]["duration_ms"], json!(1234));
|
||||
assert_eq!(payload["event_params"]["input_tokens"], json!(null));
|
||||
assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null));
|
||||
assert_eq!(payload["event_params"]["output_tokens"], json!(null));
|
||||
assert_eq!(
|
||||
payload["event_params"]["reasoning_output_tokens"],
|
||||
json!(null)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(null));
|
||||
}
|
||||
|
||||
fn sample_plugin_metadata() -> PluginTelemetryMetadata {
|
||||
|
||||
@@ -34,6 +34,7 @@ use codex_app_server_protocol::CodexErrorInfo as AppServerCodexErrorInfo;
|
||||
use codex_app_server_protocol::InitializeParams;
|
||||
use codex_app_server_protocol::RequestId;
|
||||
use codex_app_server_protocol::ServerNotification;
|
||||
use codex_app_server_protocol::TokenUsageBreakdown;
|
||||
use codex_app_server_protocol::UserInput;
|
||||
use codex_git_utils::collect_git_info;
|
||||
use codex_git_utils::get_git_repo_root;
|
||||
@@ -83,6 +84,7 @@ struct TurnState {
|
||||
thread_id: Option<String>,
|
||||
num_input_images: Option<usize>,
|
||||
resolved_config: Option<TurnResolvedConfigFact>,
|
||||
token_usage: Option<TokenUsageBreakdown>,
|
||||
created_at: Option<u64>,
|
||||
completed: Option<CompletedTurnState>,
|
||||
}
|
||||
@@ -206,6 +208,7 @@ impl AnalyticsReducer {
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
token_usage: None,
|
||||
created_at: None,
|
||||
completed: None,
|
||||
});
|
||||
@@ -358,6 +361,7 @@ impl AnalyticsReducer {
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
token_usage: None,
|
||||
created_at: None,
|
||||
completed: None,
|
||||
});
|
||||
@@ -382,11 +386,27 @@ impl AnalyticsReducer {
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
token_usage: None,
|
||||
created_at: None,
|
||||
completed: None,
|
||||
});
|
||||
turn_state.created_at = u64::try_from(notification.created_at).ok();
|
||||
}
|
||||
ServerNotification::ThreadTokenUsageUpdated(notification) => {
|
||||
let turn_state =
|
||||
self.turns
|
||||
.entry(notification.turn_id.clone())
|
||||
.or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
token_usage: None,
|
||||
created_at: None,
|
||||
completed: None,
|
||||
});
|
||||
turn_state.token_usage = Some(notification.token_usage.last);
|
||||
}
|
||||
ServerNotification::TurnCompleted(notification) => {
|
||||
let turn_state =
|
||||
self.turns
|
||||
@@ -396,6 +416,7 @@ impl AnalyticsReducer {
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
token_usage: None,
|
||||
created_at: None,
|
||||
completed: None,
|
||||
});
|
||||
@@ -453,18 +474,13 @@ impl AnalyticsReducer {
|
||||
let Some(turn_state) = self.turns.get(turn_id) else {
|
||||
return;
|
||||
};
|
||||
let Some(thread_id) = turn_state.thread_id.clone() else {
|
||||
if turn_state.thread_id.is_none()
|
||||
|| turn_state.num_input_images.is_none()
|
||||
|| turn_state.resolved_config.is_none()
|
||||
|| turn_state.completed.is_none()
|
||||
{
|
||||
return;
|
||||
};
|
||||
let Some(num_input_images) = turn_state.num_input_images else {
|
||||
return;
|
||||
};
|
||||
let Some(resolved_config) = turn_state.resolved_config.clone() else {
|
||||
return;
|
||||
};
|
||||
let Some(completed) = turn_state.completed.clone() else {
|
||||
return;
|
||||
};
|
||||
}
|
||||
let product_client_id = turn_state
|
||||
.connection_id
|
||||
.and_then(|connection_id| self.connections.get(&connection_id))
|
||||
@@ -475,12 +491,8 @@ impl AnalyticsReducer {
|
||||
event_type: "codex_turn_event",
|
||||
event_params: codex_turn_event_params(
|
||||
product_client_id,
|
||||
thread_id,
|
||||
turn_id.to_string(),
|
||||
num_input_images,
|
||||
resolved_config,
|
||||
completed,
|
||||
turn_state.created_at,
|
||||
turn_state,
|
||||
),
|
||||
},
|
||||
)));
|
||||
@@ -490,13 +502,19 @@ impl AnalyticsReducer {
|
||||
|
||||
fn codex_turn_event_params(
|
||||
product_client_id: String,
|
||||
thread_id: String,
|
||||
turn_id: String,
|
||||
num_input_images: usize,
|
||||
resolved_config: TurnResolvedConfigFact,
|
||||
completed: CompletedTurnState,
|
||||
created_at: Option<u64>,
|
||||
turn_state: &TurnState,
|
||||
) -> CodexTurnEventParams {
|
||||
let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = (
|
||||
turn_state.thread_id.clone(),
|
||||
turn_state.num_input_images,
|
||||
turn_state.resolved_config.clone(),
|
||||
turn_state.completed.clone(),
|
||||
) else {
|
||||
unreachable!("turn event params require a fully populated turn state");
|
||||
};
|
||||
let token_usage = turn_state.token_usage.clone();
|
||||
let created_at = turn_state.created_at;
|
||||
let TurnResolvedConfigFact {
|
||||
turn_id: _resolved_turn_id,
|
||||
thread_id: _resolved_thread_id,
|
||||
@@ -546,11 +564,13 @@ fn codex_turn_event_params(
|
||||
subagent_tool_call_count: None,
|
||||
web_search_count: None,
|
||||
image_generation_count: None,
|
||||
input_tokens: None,
|
||||
cached_input_tokens: None,
|
||||
output_tokens: None,
|
||||
reasoning_output_tokens: None,
|
||||
total_tokens: None,
|
||||
input_tokens: token_usage.as_ref().map(|usage| usage.input_tokens),
|
||||
cached_input_tokens: token_usage.as_ref().map(|usage| usage.cached_input_tokens),
|
||||
output_tokens: token_usage.as_ref().map(|usage| usage.output_tokens),
|
||||
reasoning_output_tokens: token_usage
|
||||
.as_ref()
|
||||
.map(|usage| usage.reasoning_output_tokens),
|
||||
total_tokens: token_usage.as_ref().map(|usage| usage.total_tokens),
|
||||
duration_ms: completed.duration_ms,
|
||||
created_at,
|
||||
completed_at: Some(completed.completed_at),
|
||||
|
||||
@@ -309,6 +309,84 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> {
|
||||
assert!(event["event_params"]["created_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["completed_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
|
||||
assert_eq!(event["event_params"]["input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["cached_input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["total_tokens"], 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn turn_start_tracks_turn_event_analytics_for_empty_input() -> Result<()> {
|
||||
let responses = vec![create_final_assistant_message_sse_response("Done")?];
|
||||
let server = create_mock_responses_server_sequence_unchecked(responses).await;
|
||||
|
||||
let codex_home = TempDir::new()?;
|
||||
write_mock_responses_config_toml_with_chatgpt_base_url(
|
||||
codex_home.path(),
|
||||
&server.uri(),
|
||||
&server.uri(),
|
||||
)?;
|
||||
enable_analytics_capture(&server, codex_home.path()).await?;
|
||||
|
||||
let mut mcp = McpProcess::new(codex_home.path()).await?;
|
||||
timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??;
|
||||
|
||||
let thread_req = mcp
|
||||
.send_thread_start_request(ThreadStartParams {
|
||||
model: Some("mock-model".to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let thread_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(thread_req)),
|
||||
)
|
||||
.await??;
|
||||
let ThreadStartResponse { thread, .. } = to_response::<ThreadStartResponse>(thread_resp)?;
|
||||
|
||||
let turn_req = mcp
|
||||
.send_turn_start_request(TurnStartParams {
|
||||
thread_id: thread.id.clone(),
|
||||
input: Vec::new(),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let turn_resp: JSONRPCResponse = timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_response_message(RequestId::Integer(turn_req)),
|
||||
)
|
||||
.await??;
|
||||
let TurnStartResponse { turn } = to_response::<TurnStartResponse>(turn_resp)?;
|
||||
|
||||
timeout(
|
||||
DEFAULT_READ_TIMEOUT,
|
||||
mcp.read_stream_until_notification_message("turn/completed"),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?;
|
||||
assert_eq!(event["event_params"]["thread_id"], thread.id);
|
||||
assert_eq!(event["event_params"]["turn_id"], turn.id);
|
||||
assert_eq!(
|
||||
event["event_params"]["product_client_id"],
|
||||
DEFAULT_CLIENT_NAME
|
||||
);
|
||||
assert_eq!(event["event_params"]["model"], "mock-model");
|
||||
assert_eq!(event["event_params"]["model_provider"], "mock_provider");
|
||||
assert_eq!(event["event_params"]["sandbox_policy"], "read_only");
|
||||
assert_eq!(event["event_params"]["num_input_images"], 0);
|
||||
assert_eq!(event["event_params"]["status"], "completed");
|
||||
assert!(event["event_params"]["created_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["completed_at"].as_u64().is_some());
|
||||
assert!(event["event_params"]["duration_ms"].as_u64().is_some());
|
||||
assert_eq!(event["event_params"]["input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["cached_input_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["reasoning_output_tokens"], 0);
|
||||
assert_eq!(event["event_params"]["total_tokens"], 0);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+29
-31
@@ -5877,37 +5877,35 @@ pub(crate) async fn run_turn(
|
||||
.await;
|
||||
}
|
||||
|
||||
if !input.is_empty() {
|
||||
let is_first_turn = {
|
||||
let mut state = sess.state.lock().await;
|
||||
state.take_next_turn_is_first()
|
||||
};
|
||||
sess.services
|
||||
.analytics_events_client
|
||||
.track_turn_resolved_config(TurnResolvedConfigFact {
|
||||
turn_id: tracking.turn_id.clone(),
|
||||
thread_id: tracking.thread_id.clone(),
|
||||
num_input_images: input
|
||||
.iter()
|
||||
.filter(|item| {
|
||||
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
|
||||
})
|
||||
.count(),
|
||||
submission_type: None,
|
||||
model: turn_context.model_info.slug.clone(),
|
||||
model_provider: turn_context.config.model_provider_id.clone(),
|
||||
sandbox_policy: turn_context.sandbox_policy.get().clone(),
|
||||
reasoning_effort: turn_context.reasoning_effort,
|
||||
reasoning_summary: Some(turn_context.reasoning_summary),
|
||||
service_tier: turn_context.config.service_tier,
|
||||
approval_policy: turn_context.approval_policy.value(),
|
||||
approvals_reviewer: turn_context.config.approvals_reviewer,
|
||||
sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(),
|
||||
collaboration_mode: turn_context.collaboration_mode.mode,
|
||||
personality: turn_context.personality,
|
||||
is_first_turn,
|
||||
});
|
||||
}
|
||||
let is_first_turn = {
|
||||
let mut state = sess.state.lock().await;
|
||||
state.take_next_turn_is_first()
|
||||
};
|
||||
sess.services
|
||||
.analytics_events_client
|
||||
.track_turn_resolved_config(TurnResolvedConfigFact {
|
||||
turn_id: tracking.turn_id.clone(),
|
||||
thread_id: tracking.thread_id.clone(),
|
||||
num_input_images: input
|
||||
.iter()
|
||||
.filter(|item| {
|
||||
matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. })
|
||||
})
|
||||
.count(),
|
||||
submission_type: None,
|
||||
model: turn_context.model_info.slug.clone(),
|
||||
model_provider: turn_context.config.model_provider_id.clone(),
|
||||
sandbox_policy: turn_context.sandbox_policy.get().clone(),
|
||||
reasoning_effort: turn_context.reasoning_effort,
|
||||
reasoning_summary: Some(turn_context.reasoning_summary),
|
||||
service_tier: turn_context.config.service_tier,
|
||||
approval_policy: turn_context.approval_policy.value(),
|
||||
approvals_reviewer: turn_context.config.approvals_reviewer,
|
||||
sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(),
|
||||
collaboration_mode: turn_context.collaboration_mode.mode,
|
||||
personality: turn_context.personality,
|
||||
is_first_turn,
|
||||
});
|
||||
|
||||
let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref());
|
||||
sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token())
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
#![cfg(not(target_os = "windows"))]
|
||||
|
||||
use anyhow::Ok;
|
||||
use codex_core::CodexAuth;
|
||||
use codex_protocol::config_types::CollaborationMode;
|
||||
use codex_protocol::config_types::ModeKind;
|
||||
use codex_protocol::config_types::Personality;
|
||||
use codex_protocol::config_types::ReasoningSummary;
|
||||
use codex_protocol::config_types::ServiceTier;
|
||||
use codex_protocol::config_types::Settings;
|
||||
use codex_protocol::items::AgentMessageContent;
|
||||
use codex_protocol::items::TurnItem;
|
||||
use codex_protocol::models::WebSearchAction;
|
||||
use codex_protocol::openai_models::ReasoningEffort;
|
||||
use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ItemCompletedEvent;
|
||||
use codex_protocol::protocol::ItemStartedEvent;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_protocol::user_input::ByteRange;
|
||||
use codex_protocol::user_input::TextElement;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
@@ -42,11 +35,8 @@ use core_test_support::test_codex::test_codex;
|
||||
use core_test_support::wait_for_event;
|
||||
use core_test_support::wait_for_event_match;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::Value;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
fn image_generation_artifact_path(codex_home: &Path, session_id: &str, call_id: &str) -> PathBuf {
|
||||
fn sanitize(value: &str) -> String {
|
||||
@@ -130,217 +120,6 @@ async fn user_message_item_is_emitted() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn user_turn_tracks_turn_metadata_analytics() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
mount_sse_once(
|
||||
&server,
|
||||
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let chatgpt_base_url = server.uri();
|
||||
let mut builder = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(move |config| {
|
||||
config.chatgpt_base_url = chatgpt_base_url;
|
||||
config.personality = Some(Personality::None);
|
||||
});
|
||||
let TestCodex {
|
||||
codex,
|
||||
session_configured,
|
||||
config,
|
||||
..
|
||||
} = builder.build(&server).await?;
|
||||
|
||||
codex
|
||||
.submit(Op::UserTurn {
|
||||
items: vec![UserInput::Text {
|
||||
text: "hello turn metadata analytics".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
cwd: config.cwd.to_path_buf(),
|
||||
approval_policy: AskForApproval::Never,
|
||||
approvals_reviewer: None,
|
||||
sandbox_policy: SandboxPolicy::new_read_only_policy(),
|
||||
model: session_configured.model.clone(),
|
||||
effort: Some(ReasoningEffort::High),
|
||||
summary: Some(ReasoningSummary::Detailed),
|
||||
service_tier: Some(Some(ServiceTier::Flex)),
|
||||
collaboration_mode: Some(CollaborationMode {
|
||||
mode: ModeKind::Plan,
|
||||
settings: Settings {
|
||||
model: session_configured.model.clone(),
|
||||
reasoning_effort: Some(ReasoningEffort::High),
|
||||
developer_instructions: None,
|
||||
},
|
||||
}),
|
||||
personality: None,
|
||||
})
|
||||
.await?;
|
||||
|
||||
wait_for_event(&codex, |event| matches!(event, EventMsg::TurnComplete(_))).await;
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let event = loop {
|
||||
let requests = server.received_requests().await.unwrap_or_default();
|
||||
let turn_events = requests
|
||||
.into_iter()
|
||||
.filter(|request| request.url.path() == "/codex/analytics-events/events")
|
||||
.filter_map(|request| serde_json::from_slice::<Value>(&request.body).ok())
|
||||
.flat_map(|payload| {
|
||||
payload["events"]
|
||||
.as_array()
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
})
|
||||
.filter(|event| event["event_type"] == "codex_turn_event")
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(event) = turn_events.last().cloned() {
|
||||
break event;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!("timed out waiting for turn analytics event");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
|
||||
let event_params = &event["event_params"];
|
||||
|
||||
assert_eq!(event_params["sandbox_policy"], "read_only");
|
||||
assert_eq!(
|
||||
event_params["product_client_id"],
|
||||
serde_json::json!(codex_core::default_client::originator().value)
|
||||
);
|
||||
assert_eq!(event_params["model"], session_configured.model);
|
||||
assert_eq!(event_params["model_provider"], config.model_provider_id);
|
||||
assert_eq!(event_params["reasoning_effort"], "high");
|
||||
assert_eq!(event_params["reasoning_summary"], "detailed");
|
||||
assert_eq!(event_params["service_tier"], "flex");
|
||||
assert_eq!(event_params["approval_policy"], "never");
|
||||
assert_eq!(
|
||||
event_params["approvals_reviewer"],
|
||||
config.approvals_reviewer.to_string()
|
||||
);
|
||||
assert_eq!(
|
||||
event_params["sandbox_network_access"],
|
||||
config.permissions.network_sandbox_policy.is_enabled()
|
||||
);
|
||||
assert_eq!(event_params["collaboration_mode"], "plan");
|
||||
assert!(event_params["personality"].is_null());
|
||||
assert_eq!(event_params["num_input_images"], 0);
|
||||
assert_eq!(event_params["is_first_turn"], true);
|
||||
assert!(event_params["thread_id"].as_str().is_some());
|
||||
assert!(event_params["turn_id"].as_str().is_some());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn resumed_thread_turn_tracks_is_not_first_turn_analytics() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = start_mock_server().await;
|
||||
mount_sse_once(
|
||||
&server,
|
||||
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
|
||||
)
|
||||
.await;
|
||||
mount_sse_once(
|
||||
&server,
|
||||
sse(vec![ev_response_created("resp-2"), ev_completed("resp-2")]),
|
||||
)
|
||||
.await;
|
||||
|
||||
let chatgpt_base_url = server.uri();
|
||||
let mut builder = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(move |config| {
|
||||
config.chatgpt_base_url = chatgpt_base_url;
|
||||
});
|
||||
let initial = builder.build(&server).await?;
|
||||
|
||||
initial
|
||||
.codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "first turn".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&initial.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let rollout_path = initial
|
||||
.session_configured
|
||||
.rollout_path
|
||||
.clone()
|
||||
.expect("persisted rollout path should be present");
|
||||
let home = initial.home.clone();
|
||||
let resumed_chatgpt_base_url = server.uri();
|
||||
|
||||
let mut resume_builder = test_codex()
|
||||
.with_auth(CodexAuth::create_dummy_chatgpt_auth_for_testing())
|
||||
.with_config(move |config| {
|
||||
config.chatgpt_base_url = resumed_chatgpt_base_url;
|
||||
});
|
||||
let resumed = resume_builder.resume(&server, home, rollout_path).await?;
|
||||
|
||||
resumed
|
||||
.codex
|
||||
.submit(Op::UserInput {
|
||||
items: vec![UserInput::Text {
|
||||
text: "resumed turn".into(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
final_output_json_schema: None,
|
||||
})
|
||||
.await?;
|
||||
wait_for_event(&resumed.codex, |event| {
|
||||
matches!(event, EventMsg::TurnComplete(_))
|
||||
})
|
||||
.await;
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let resumed_turn_event = loop {
|
||||
let requests = server.received_requests().await.unwrap_or_default();
|
||||
let turn_events = requests
|
||||
.iter()
|
||||
.filter(|request| request.url.path() == "/codex/analytics-events/events")
|
||||
.filter_map(|request| serde_json::from_slice::<Value>(&request.body).ok())
|
||||
.flat_map(|payload| {
|
||||
payload["events"]
|
||||
.as_array()
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
})
|
||||
.filter(|event| event["event_type"] == "codex_turn_event")
|
||||
.collect::<Vec<_>>();
|
||||
if let Some(event) = turn_events.last().cloned()
|
||||
&& turn_events.len() >= 2
|
||||
{
|
||||
break event;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!("timed out waiting for resumed turn analytics event");
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
|
||||
assert_eq!(resumed_turn_event["event_params"]["is_first_turn"], false);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn assistant_message_item_is_emitted() -> anyhow::Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
Reference in New Issue
Block a user