openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-05 15:18:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[codex-analytics] add goal status at turn end to turn events

Browse Source
This commit is contained in:
Mark Steinbrick
2026-05-26 19:31:31 -07:00
parent 8d398d3c52
commit a339192d1b
9 changed files with 178 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
codex-rs/analytics/src/analytics_client_tests.rs
View File

@@ -50,6 +50,7 @@ use crate::facts::CompactionStatus;
use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::InputError;
@@ -140,6 +141,7 @@ use codex_protocol::protocol::HookRunStatus;
use codex_protocol::protocol::HookSource;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadGoalStatus;
use codex_protocol::protocol::ThreadSource;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionGrantScope;
@@ -331,6 +333,18 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag
}
}
fn sample_goal_status_at_turn_end_fact(
thread_id: &str,
turn_id: &str,
goal_status_at_turn_end: Option<ThreadGoalStatus>,
) -> GoalStatusAtTurnEndFact {
GoalStatusAtTurnEndFact {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_string(),
goal_status_at_turn_end,
}
}
fn sample_turn_completed_notification(
thread_id: &str,
turn_id: &str,
@@ -632,6 +646,15 @@ async fn ingest_turn_prerequisites(
)
.await;
}
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
sample_goal_status_at_turn_end_fact("thread-2", "turn-2", None),
))),
out,
)
.await;
}
async fn ingest_review_prerequisites(
@@ -3233,6 +3256,7 @@ fn turn_event_serializes_expected_shape() {
approvals_reviewer: "auto_review".to_string(),
sandbox_network_access: true,
collaboration_mode: Some("plan"),
goal_status_at_turn_end: None,
personality: Some("pragmatic".to_string()),
num_input_images: 2,
is_first_turn: true,
@@ -3295,6 +3319,7 @@ fn turn_event_serializes_expected_shape() {
"approvals_reviewer": "auto_review",
"sandbox_network_access": true,
"collaboration_mode": "plan",
"goal_status_at_turn_end": null,
"personality": "pragmatic",
"num_input_images": 2,
"is_first_turn": true,
@@ -3571,6 +3596,18 @@ async fn turn_lifecycle_emits_turn_event() {
/*include_token_usage*/ true,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
sample_goal_status_at_turn_end_fact(
"thread-2",
"turn-2",
Some(ThreadGoalStatus::BudgetLimited),
),
))),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Notification(Box::new(sample_turn_completed_notification(
@@ -3638,6 +3675,10 @@ async fn turn_lifecycle_emits_turn_event() {
json!(13)
);
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
assert_eq!(
payload["event_params"]["goal_status_at_turn_end"],
json!("budget_limited")
);
}
#[tokio::test]

7
codex-rs/analytics/src/client.rs
View File

@@ -10,6 +10,7 @@ use crate::facts::AppInvocation;
use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
@@ -256,6 +257,12 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_goal_status_at_turn_end(&self, fact: GoalStatusAtTurnEndFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(fact)),
));
}
pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput {

1
codex-rs/analytics/src/events.rs
View File

@@ -792,6 +792,7 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) approvals_reviewer: String,
pub(crate) sandbox_network_access: bool,
pub(crate) collaboration_mode: Option<&'static str>,
pub(crate) goal_status_at_turn_end: Option<&'static str>,
pub(crate) personality: Option<String>,
pub(crate) num_input_images: usize,
pub(crate) is_first_turn: bool,

9
codex-rs/analytics/src/facts.rs
View File

@@ -24,6 +24,7 @@ use codex_protocol::protocol::HookSource;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SkillScope;
use codex_protocol::protocol::SubAgentSource;
use codex_protocol::protocol::ThreadGoalStatus;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::request_permissions::RequestPermissionsResponse;
use serde::Serialize;
@@ -99,6 +100,13 @@ pub struct TurnTokenUsageFact {
pub token_usage: TokenUsage,
}
#[derive(Clone)]
pub struct GoalStatusAtTurnEndFact {
pub turn_id: String,
pub thread_id: String,
pub goal_status_at_turn_end: Option<ThreadGoalStatus>,
}
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TurnStatus {
@@ -329,6 +337,7 @@ pub(crate) enum CustomAnalyticsFact {
GuardianReview(Box<GuardianReviewEventParams>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),
GoalStatusAtTurnEnd(Box<GoalStatusAtTurnEndFact>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
AppUsed(AppUsedInput),

1
codex-rs/analytics/src/lib.rs
View File

@@ -31,6 +31,7 @@ pub use facts::CompactionReason;
pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
pub use facts::GoalStatusAtTurnEndFact;
pub use facts::HookRunFact;
pub use facts::InputError;
pub use facts::InvocationType;

53
codex-rs/analytics/src/reducer.rs
View File

@@ -64,6 +64,7 @@ use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
@@ -118,6 +119,7 @@ use codex_protocol::config_types::ReasoningSummary;
use codex_protocol::models::PermissionProfile;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SkillScope;
use codex_protocol::protocol::ThreadGoalStatus;
use codex_protocol::protocol::ThreadSource;
use codex_protocol::protocol::TokenUsage;
use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionGrantScope;
@@ -324,6 +326,8 @@ struct TurnState {
resolved_config: Option<TurnResolvedConfigFact>,
started_at: Option<u64>,
token_usage: Option<TokenUsage>,
// Outer None means the completion fact has not arrived; inner None means no goal.
goal_status_at_turn_end: Option<Option<ThreadGoalStatus>>,
completed: Option<CompletedTurnState>,
latest_diff: Option<String>,
steer_count: usize,
@@ -464,6 +468,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::TurnTokenUsage(input) => {
self.ingest_turn_token_usage(*input, out).await;
}
CustomAnalyticsFact::GoalStatusAtTurnEnd(input) => {
self.ingest_goal_status_at_turn_end(*input, out).await;
}
CustomAnalyticsFact::SkillInvoked(input) => {
self.ingest_skill_invoked(input, out).await;
}
@@ -611,6 +618,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -635,6 +643,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -645,6 +654,30 @@ impl AnalyticsReducer {
self.maybe_emit_turn_event(&turn_id, out).await;
}
async fn ingest_goal_status_at_turn_end(
&mut self,
input: GoalStatusAtTurnEndFact,
out: &mut Vec<TrackEventRequest>,
) {
let turn_id = input.turn_id.clone();
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
connection_id: None,
thread_id: None,
num_input_images: None,
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
});
turn_state.thread_id = Some(input.thread_id);
turn_state.goal_status_at_turn_end = Some(input.goal_status_at_turn_end);
self.maybe_emit_turn_event(&turn_id, out).await;
}
async fn ingest_skill_invoked(
&mut self,
input: SkillInvokedInput,
@@ -800,6 +833,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -1159,6 +1193,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -1180,6 +1215,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -1199,6 +1235,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
@@ -1478,6 +1515,7 @@ impl AnalyticsReducer {
if turn_state.thread_id.is_none()
|| turn_state.num_input_images.is_none()
|| turn_state.resolved_config.is_none()
|| turn_state.goal_status_at_turn_end.is_none()
|| turn_state.completed.is_none()
{
return;
@@ -2482,6 +2520,10 @@ fn codex_turn_event_params(
approvals_reviewer: approvals_reviewer.to_string(),
sandbox_network_access,
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
goal_status_at_turn_end: turn_state
.goal_status_at_turn_end
.flatten()
.map(goal_status_mode),
personality: personality_mode(personality),
num_input_images,
is_first_turn,
@@ -2548,6 +2590,17 @@ fn collaboration_mode_mode(mode: ModeKind) -> &'static str {
}
}
fn goal_status_mode(status: ThreadGoalStatus) -> &'static str {
match status {
ThreadGoalStatus::Active => "active",
ThreadGoalStatus::Paused => "paused",
ThreadGoalStatus::Blocked => "blocked",
ThreadGoalStatus::UsageLimited => "usage_limited",
ThreadGoalStatus::BudgetLimited => "budget_limited",
ThreadGoalStatus::Complete => "complete",
}
}
fn reasoning_summary_mode(summary: Option<ReasoningSummary>) -> Option<String> {
match summary {
Some(ReasoningSummary::None) | None => None,

31
codex-rs/core/src/goals.rs
View File

@@ -431,6 +431,37 @@ impl Session {
.map(|goal| goal.map(protocol_goal_from_state))
}
pub(crate) async fn goal_status_at_turn_end(
&self,
turn_context: &TurnContext,
) -> Option<ThreadGoalStatus> {
if !self.enabled(Feature::Goals)
|| should_ignore_goal_for_mode(turn_context.collaboration_mode.mode)
{
return None;
}
let state_db = match self.state_db_for_thread_goals().await {
Ok(Some(state_db)) => state_db,
Ok(None) => return None,
Err(err) => {
tracing::warn!("failed to open state db at turn end: {err}");
return None;
}
};
match state_db
.thread_goals()
.get_thread_goal(self.conversation_id)
.await
{
Ok(Some(goal)) => Some(protocol_goal_status_from_state(goal.status)),
Ok(None) => None,
Err(err) => {
tracing::warn!("failed to read thread goal at turn end: {err}");
None
}
}
}
pub(crate) async fn set_thread_goal(
&self,
turn_context: &TurnContext,

8
codex-rs/core/src/session/tests.rs
View File

@@ -8743,6 +8743,10 @@ async fn create_thread_goal_fills_empty_thread_preview() -> anyhow::Result<()> {
Some("Keep improving the benchmark"),
page.items[0].preview.as_deref()
);
assert_eq!(
Some(ThreadGoalStatus::Active),
sess.goal_status_at_turn_end(tc.as_ref()).await
);
Ok(())
}
@@ -8823,6 +8827,10 @@ async fn budget_limited_accounting_steers_active_turn_without_aborting() -> anyh
.expect("goal should remain persisted after accounting");
assert_eq!(codex_state::ThreadGoalStatus::BudgetLimited, goal.status);
assert_eq!(25, goal.tokens_used);
assert_eq!(
Some(ThreadGoalStatus::BudgetLimited),
sess.goal_status_at_turn_end(tc.as_ref()).await
);
set_total_token_usage(
&sess,

31
codex-rs/core/src/tasks/mod.rs
View File

@@ -33,6 +33,7 @@ use crate::session::turn_context::TurnContext;
use crate::state::ActiveTurn;
use crate::state::RunningTask;
use crate::state::TaskKind;
use codex_analytics::GoalStatusAtTurnEndFact;
use codex_analytics::TurnTokenUsageFact;
use codex_login::AuthManager;
use codex_models_manager::manager::SharedModelsManager;
@@ -515,14 +516,19 @@ impl Session {
self.emit_turn_abort_lifecycle(reason.clone(), turn_context.extension_data.as_ref())
.await;
}
if (aborted_turn || reason == TurnAbortReason::Interrupted)
&& let Err(err) = self
if aborted_turn || reason == TurnAbortReason::Interrupted {
if let Err(err) = self
.goal_runtime_apply(GoalRuntimeEvent::TaskAborted {
turn_context: turn_context.as_deref(),
})
.await
{
warn!("failed to apply goal runtime abort event: {err}");
{
warn!("failed to apply goal runtime abort event: {err}");
}
if let Some(turn_context) = turn_context.as_deref() {
self.track_goal_status_at_turn_end_analytics(turn_context)
.await;
}
}
if let Some(active_turn) = active_turn_to_clear {
// Let interrupted tasks observe cancellation before dropping pending approvals, or an
@@ -572,6 +578,10 @@ impl Session {
{
warn!("failed to apply goal runtime abort event: {err}");
}
if let Some(turn_context) = turn_context.as_deref() {
self.track_goal_status_at_turn_end_analytics(turn_context)
.await;
}
// Let interrupted tasks observe cancellation before dropping pending approvals, or an
// in-flight approval wait can surface as a model-visible rejection before TurnAborted.
self.input_queue.clear_pending(&active_turn).await;
@@ -774,6 +784,8 @@ impl Session {
{
warn!("failed to apply goal runtime turn-finished event: {err}");
}
self.track_goal_status_at_turn_end_analytics(turn_context.as_ref())
.await;
let event = EventMsg::TurnComplete(TurnCompleteEvent {
turn_id: turn_context.sub_id.clone(),
last_agent_message,
@@ -811,6 +823,17 @@ impl Session {
}
}
async fn track_goal_status_at_turn_end_analytics(&self, turn_context: &TurnContext) {
let goal_status_at_turn_end = self.goal_status_at_turn_end(turn_context).await;
self.services
.analytics_events_client
.track_goal_status_at_turn_end(GoalStatusAtTurnEndFact {
turn_id: turn_context.sub_id.clone(),
thread_id: self.conversation_id.to_string(),
goal_status_at_turn_end,
});
}
async fn take_active_turn(&self) -> Option<ActiveTurn> {
let mut active = self.active_turn.lock().await;
active.take()