From bb1770753d10b3fcf4224447309329bb2e542473 Mon Sep 17 00:00:00 2001 From: Mark Steinbrick Date: Wed, 27 May 2026 13:41:00 -0700 Subject: [PATCH] add goal_status_at_turn_start --- .../analytics/src/analytics_client_tests.rs | 39 ++++++++++++++++ codex-rs/analytics/src/client.rs | 7 +++ codex-rs/analytics/src/events.rs | 1 + codex-rs/analytics/src/facts.rs | 8 ++++ codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/reducer.rs | 44 ++++++++++++++++++- codex-rs/core/src/goals.rs | 27 +++++++++--- 7 files changed, 119 insertions(+), 8 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 48637b2ad3..dd270fd4cc 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -51,6 +51,7 @@ use crate::facts::CompactionStrategy; use crate::facts::CompactionTrigger; use crate::facts::CustomAnalyticsFact; use crate::facts::GoalStatusAtTurnEndFact; +use crate::facts::GoalStatusAtTurnStartFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; use crate::facts::InputError; @@ -345,6 +346,18 @@ fn sample_goal_status_at_turn_end_fact( } } +fn sample_goal_status_at_turn_start_fact( + thread_id: &str, + turn_id: &str, + goal_status_at_turn_start: Option, +) -> GoalStatusAtTurnStartFact { + GoalStatusAtTurnStartFact { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_string(), + goal_status_at_turn_start, + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -647,6 +660,14 @@ async fn ingest_turn_prerequisites( .await; } + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new( + sample_goal_status_at_turn_start_fact("thread-2", "turn-2", None), + ))), + out, + ) + .await; reducer .ingest( AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new( @@ -3256,6 +3277,7 @@ fn turn_event_serializes_expected_shape() { approvals_reviewer: "auto_review".to_string(), sandbox_network_access: true, collaboration_mode: Some("plan"), + goal_status_at_turn_start: None, goal_status_at_turn_end: None, personality: Some("pragmatic".to_string()), num_input_images: 2, @@ -3319,6 +3341,7 @@ fn turn_event_serializes_expected_shape() { "approvals_reviewer": "auto_review", "sandbox_network_access": true, "collaboration_mode": "plan", + "goal_status_at_turn_start": null, "goal_status_at_turn_end": null, "personality": "pragmatic", "num_input_images": 2, @@ -3596,6 +3619,18 @@ async fn turn_lifecycle_emits_turn_event() { /*include_token_usage*/ true, ) .await; + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new( + sample_goal_status_at_turn_start_fact( + "thread-2", + "turn-2", + Some(ThreadGoalStatus::Active), + ), + ))), + &mut out, + ) + .await; reducer .ingest( AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new( @@ -3675,6 +3710,10 @@ async fn turn_lifecycle_emits_turn_event() { json!(13) ); assert_eq!(payload["event_params"]["total_tokens"], json!(321)); + assert_eq!( + payload["event_params"]["goal_status_at_turn_start"], + json!("active") + ); assert_eq!( payload["event_params"]["goal_status_at_turn_end"], json!("budget_limited") diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 0f07a3136b..679504c115 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -11,6 +11,7 @@ use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CustomAnalyticsFact; use crate::facts::GoalStatusAtTurnEndFact; +use crate::facts::GoalStatusAtTurnStartFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; use crate::facts::PluginState; @@ -257,6 +258,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_goal_status_at_turn_start(&self, fact: GoalStatusAtTurnStartFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(fact)), + )); + } + pub fn track_goal_status_at_turn_end(&self, fact: GoalStatusAtTurnEndFact) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(fact)), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index a7d406cc17..6e1ac147e5 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -792,6 +792,7 @@ pub(crate) struct CodexTurnEventParams { pub(crate) approvals_reviewer: String, pub(crate) sandbox_network_access: bool, pub(crate) collaboration_mode: Option<&'static str>, + pub(crate) goal_status_at_turn_start: Option<&'static str>, pub(crate) goal_status_at_turn_end: Option<&'static str>, pub(crate) personality: Option, pub(crate) num_input_images: usize, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index baf81fe273..dad9705bd9 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -100,6 +100,13 @@ pub struct TurnTokenUsageFact { pub token_usage: TokenUsage, } +#[derive(Clone)] +pub struct GoalStatusAtTurnStartFact { + pub turn_id: String, + pub thread_id: String, + pub goal_status_at_turn_start: Option, +} + #[derive(Clone)] pub struct GoalStatusAtTurnEndFact { pub turn_id: String, @@ -337,6 +344,7 @@ pub(crate) enum CustomAnalyticsFact { GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), + GoalStatusAtTurnStart(Box), GoalStatusAtTurnEnd(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 9118477169..2585ccb22e 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -32,6 +32,7 @@ pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; pub use facts::GoalStatusAtTurnEndFact; +pub use facts::GoalStatusAtTurnStartFact; pub use facts::HookRunFact; pub use facts::InputError; pub use facts::InvocationType; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 5f8812531c..24693dccd0 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -65,6 +65,7 @@ use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; use crate::facts::CustomAnalyticsFact; use crate::facts::GoalStatusAtTurnEndFact; +use crate::facts::GoalStatusAtTurnStartFact; use crate::facts::HookRunInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -326,7 +327,8 @@ struct TurnState { resolved_config: Option, started_at: Option, token_usage: Option, - // Outer None means the completion fact has not arrived; inner None means no goal. + // `None` means a boundary fact has not arrived; `Some(None)` means no applicable goal. + goal_status_at_turn_start: Option>, goal_status_at_turn_end: Option>, completed: Option, latest_diff: Option, @@ -468,6 +470,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnTokenUsage(input) => { self.ingest_turn_token_usage(*input, out).await; } + CustomAnalyticsFact::GoalStatusAtTurnStart(input) => { + self.ingest_goal_status_at_turn_start(*input, out).await; + } CustomAnalyticsFact::GoalStatusAtTurnEnd(input) => { self.ingest_goal_status_at_turn_end(*input, out).await; } @@ -618,6 +623,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -643,6 +649,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -654,6 +661,31 @@ impl AnalyticsReducer { self.maybe_emit_turn_event(&turn_id, out).await; } + async fn ingest_goal_status_at_turn_start( + &mut self, + input: GoalStatusAtTurnStartFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + token_usage: None, + goal_status_at_turn_start: None, + goal_status_at_turn_end: None, + completed: None, + latest_diff: None, + steer_count: 0, + tool_counts: TurnToolCounts::default(), + }); + turn_state.thread_id = Some(input.thread_id); + turn_state.goal_status_at_turn_start = Some(input.goal_status_at_turn_start); + self.maybe_emit_turn_event(&turn_id, out).await; + } + async fn ingest_goal_status_at_turn_end( &mut self, input: GoalStatusAtTurnEndFact, @@ -667,6 +699,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -833,6 +866,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -1193,6 +1227,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -1215,6 +1250,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -1235,6 +1271,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_start: None, goal_status_at_turn_end: None, completed: None, latest_diff: None, @@ -1515,6 +1552,7 @@ impl AnalyticsReducer { if turn_state.thread_id.is_none() || turn_state.num_input_images.is_none() || turn_state.resolved_config.is_none() + || turn_state.goal_status_at_turn_start.is_none() || turn_state.goal_status_at_turn_end.is_none() || turn_state.completed.is_none() { @@ -2520,6 +2558,10 @@ fn codex_turn_event_params( approvals_reviewer: approvals_reviewer.to_string(), sandbox_network_access, collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), + goal_status_at_turn_start: turn_state + .goal_status_at_turn_start + .flatten() + .map(goal_status_mode), goal_status_at_turn_end: turn_state .goal_status_at_turn_end .flatten() diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index dca457667d..88a69b6a06 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -15,6 +15,7 @@ use crate::state::TurnState; use crate::tasks::RegularTask; use crate::tools::handlers::goal_spec::UPDATE_GOAL_TOOL_NAME; use anyhow::Context; +use codex_analytics::GoalStatusAtTurnStartFact; use codex_features::Feature; use codex_otel::GOAL_BLOCKED_METRIC; use codex_otel::GOAL_BUDGET_LIMITED_METRIC; @@ -347,8 +348,16 @@ impl Session { turn_context, token_usage, } => Box::pin(async move { - self.mark_thread_goal_turn_started(turn_context, token_usage) + let goal_status_at_turn_start = self + .mark_thread_goal_turn_started(turn_context, token_usage) .await; + self.services + .analytics_events_client + .track_goal_status_at_turn_start(GoalStatusAtTurnStartFact { + turn_id: turn_context.sub_id.clone(), + thread_id: self.conversation_id.to_string(), + goal_status_at_turn_start, + }); Ok(()) }), GoalRuntimeEvent::ToolCompleted { @@ -874,25 +883,25 @@ impl Session { &self, turn_context: &TurnContext, token_usage: TokenUsage, - ) { + ) -> Option { self.goal_runtime.accounting.lock().await.turn = Some(GoalTurnAccountingSnapshot::new( turn_context.sub_id.clone(), token_usage, )); if !self.enabled(Feature::Goals) { - return; + return None; } if should_ignore_goal_for_mode(turn_context.collaboration_mode.mode) { self.clear_active_goal_accounting(turn_context).await; - return; + return None; } let state_db = match self.state_db_for_thread_goals().await { Ok(Some(state_db)) => state_db, - Ok(None) => return, + Ok(None) => return None, Err(err) => { tracing::warn!("failed to open state db at turn start: {err}"); - return; + return None; } }; match state_db @@ -907,6 +916,7 @@ impl Session { | codex_state::ThreadGoalStatus::BudgetLimited ) => { + let status = protocol_goal_status_from_state(goal.status); let mut accounting = self.goal_runtime.accounting.lock().await; if let Some(turn) = accounting.turn.as_mut() && turn.turn_id == turn_context.sub_id @@ -914,17 +924,20 @@ impl Session { turn.mark_active_goal(goal.goal_id.clone()); } accounting.wall_clock.mark_active_goal(goal.goal_id); + Some(status) } - Ok(Some(_)) | Ok(None) => { + Ok(goal) => { self.goal_runtime .accounting .lock() .await .wall_clock .clear_active_goal(); + goal.map(|goal| protocol_goal_status_from_state(goal.status)) } Err(err) => { tracing::warn!("failed to read thread goal at turn start: {err}"); + None } } }