diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index fa64b46d7a..48637b2ad3 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -50,6 +50,7 @@ use crate::facts::CompactionStatus; use crate::facts::CompactionStrategy; use crate::facts::CompactionTrigger; use crate::facts::CustomAnalyticsFact; +use crate::facts::GoalStatusAtTurnEndFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; use crate::facts::InputError; @@ -140,6 +141,7 @@ use codex_protocol::protocol::HookRunStatus; use codex_protocol::protocol::HookSource; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsage; use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionGrantScope; @@ -331,6 +333,18 @@ fn sample_turn_token_usage_fact(thread_id: &str, turn_id: &str) -> TurnTokenUsag } } +fn sample_goal_status_at_turn_end_fact( + thread_id: &str, + turn_id: &str, + goal_status_at_turn_end: Option, +) -> GoalStatusAtTurnEndFact { + GoalStatusAtTurnEndFact { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_string(), + goal_status_at_turn_end, + } +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -632,6 +646,15 @@ async fn ingest_turn_prerequisites( ) .await; } + + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new( + sample_goal_status_at_turn_end_fact("thread-2", "turn-2", None), + ))), + out, + ) + .await; } async fn ingest_review_prerequisites( @@ -3233,6 +3256,7 @@ fn turn_event_serializes_expected_shape() { approvals_reviewer: "auto_review".to_string(), sandbox_network_access: true, collaboration_mode: Some("plan"), + goal_status_at_turn_end: None, personality: Some("pragmatic".to_string()), num_input_images: 2, is_first_turn: true, @@ -3295,6 +3319,7 @@ fn turn_event_serializes_expected_shape() { "approvals_reviewer": "auto_review", "sandbox_network_access": true, "collaboration_mode": "plan", + "goal_status_at_turn_end": null, "personality": "pragmatic", "num_input_images": 2, "is_first_turn": true, @@ -3571,6 +3596,18 @@ async fn turn_lifecycle_emits_turn_event() { /*include_token_usage*/ true, ) .await; + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new( + sample_goal_status_at_turn_end_fact( + "thread-2", + "turn-2", + Some(ThreadGoalStatus::BudgetLimited), + ), + ))), + &mut out, + ) + .await; reducer .ingest( AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( @@ -3638,6 +3675,10 @@ async fn turn_lifecycle_emits_turn_event() { json!(13) ); assert_eq!(payload["event_params"]["total_tokens"], json!(321)); + assert_eq!( + payload["event_params"]["goal_status_at_turn_end"], + json!("budget_limited") + ); } #[tokio::test] diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index fbcfa32dc5..0f07a3136b 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -10,6 +10,7 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CustomAnalyticsFact; +use crate::facts::GoalStatusAtTurnEndFact; use crate::facts::HookRunFact; use crate::facts::HookRunInput; use crate::facts::PluginState; @@ -256,6 +257,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_goal_status_at_turn_end(&self, fact: GoalStatusAtTurnEndFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(fact)), + )); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 8791e5767a..a7d406cc17 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -792,6 +792,7 @@ pub(crate) struct CodexTurnEventParams { pub(crate) approvals_reviewer: String, pub(crate) sandbox_network_access: bool, pub(crate) collaboration_mode: Option<&'static str>, + pub(crate) goal_status_at_turn_end: Option<&'static str>, pub(crate) personality: Option, pub(crate) num_input_images: usize, pub(crate) is_first_turn: bool, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 9a85276712..baf81fe273 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -24,6 +24,7 @@ use codex_protocol::protocol::HookSource; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use codex_protocol::protocol::SubAgentSource; +use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::TokenUsage; use codex_protocol::request_permissions::RequestPermissionsResponse; use serde::Serialize; @@ -99,6 +100,13 @@ pub struct TurnTokenUsageFact { pub token_usage: TokenUsage, } +#[derive(Clone)] +pub struct GoalStatusAtTurnEndFact { + pub turn_id: String, + pub thread_id: String, + pub goal_status_at_turn_end: Option, +} + #[derive(Clone, Copy, Debug, Serialize)] #[serde(rename_all = "snake_case")] pub enum TurnStatus { @@ -329,6 +337,7 @@ pub(crate) enum CustomAnalyticsFact { GuardianReview(Box), TurnResolvedConfig(Box), TurnTokenUsage(Box), + GoalStatusAtTurnEnd(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index a33ca7b9e3..9118477169 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -31,6 +31,7 @@ pub use facts::CompactionReason; pub use facts::CompactionStatus; pub use facts::CompactionStrategy; pub use facts::CompactionTrigger; +pub use facts::GoalStatusAtTurnEndFact; pub use facts::HookRunFact; pub use facts::InputError; pub use facts::InvocationType; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index f2f036b8f4..5f8812531c 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -64,6 +64,7 @@ use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::CodexCompactionEvent; use crate::facts::CustomAnalyticsFact; +use crate::facts::GoalStatusAtTurnEndFact; use crate::facts::HookRunInput; use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; @@ -118,6 +119,7 @@ use codex_protocol::config_types::ReasoningSummary; use codex_protocol::models::PermissionProfile; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; +use codex_protocol::protocol::ThreadGoalStatus; use codex_protocol::protocol::ThreadSource; use codex_protocol::protocol::TokenUsage; use codex_protocol::request_permissions::PermissionGrantScope as CorePermissionGrantScope; @@ -324,6 +326,8 @@ struct TurnState { resolved_config: Option, started_at: Option, token_usage: Option, + // Outer None means the completion fact has not arrived; inner None means no goal. + goal_status_at_turn_end: Option>, completed: Option, latest_diff: Option, steer_count: usize, @@ -464,6 +468,9 @@ impl AnalyticsReducer { CustomAnalyticsFact::TurnTokenUsage(input) => { self.ingest_turn_token_usage(*input, out).await; } + CustomAnalyticsFact::GoalStatusAtTurnEnd(input) => { + self.ingest_goal_status_at_turn_end(*input, out).await; + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -611,6 +618,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -635,6 +643,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -645,6 +654,30 @@ impl AnalyticsReducer { self.maybe_emit_turn_event(&turn_id, out).await; } + async fn ingest_goal_status_at_turn_end( + &mut self, + input: GoalStatusAtTurnEndFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + started_at: None, + token_usage: None, + goal_status_at_turn_end: None, + completed: None, + latest_diff: None, + steer_count: 0, + tool_counts: TurnToolCounts::default(), + }); + turn_state.thread_id = Some(input.thread_id); + turn_state.goal_status_at_turn_end = Some(input.goal_status_at_turn_end); + self.maybe_emit_turn_event(&turn_id, out).await; + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -800,6 +833,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -1159,6 +1193,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -1180,6 +1215,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -1199,6 +1235,7 @@ impl AnalyticsReducer { resolved_config: None, started_at: None, token_usage: None, + goal_status_at_turn_end: None, completed: None, latest_diff: None, steer_count: 0, @@ -1478,6 +1515,7 @@ impl AnalyticsReducer { if turn_state.thread_id.is_none() || turn_state.num_input_images.is_none() || turn_state.resolved_config.is_none() + || turn_state.goal_status_at_turn_end.is_none() || turn_state.completed.is_none() { return; @@ -2482,6 +2520,10 @@ fn codex_turn_event_params( approvals_reviewer: approvals_reviewer.to_string(), sandbox_network_access, collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), + goal_status_at_turn_end: turn_state + .goal_status_at_turn_end + .flatten() + .map(goal_status_mode), personality: personality_mode(personality), num_input_images, is_first_turn, @@ -2548,6 +2590,17 @@ fn collaboration_mode_mode(mode: ModeKind) -> &'static str { } } +fn goal_status_mode(status: ThreadGoalStatus) -> &'static str { + match status { + ThreadGoalStatus::Active => "active", + ThreadGoalStatus::Paused => "paused", + ThreadGoalStatus::Blocked => "blocked", + ThreadGoalStatus::UsageLimited => "usage_limited", + ThreadGoalStatus::BudgetLimited => "budget_limited", + ThreadGoalStatus::Complete => "complete", + } +} + fn reasoning_summary_mode(summary: Option) -> Option { match summary { Some(ReasoningSummary::None) | None => None, diff --git a/codex-rs/core/src/goals.rs b/codex-rs/core/src/goals.rs index c8e76f7d41..dca457667d 100644 --- a/codex-rs/core/src/goals.rs +++ b/codex-rs/core/src/goals.rs @@ -431,6 +431,37 @@ impl Session { .map(|goal| goal.map(protocol_goal_from_state)) } + pub(crate) async fn goal_status_at_turn_end( + &self, + turn_context: &TurnContext, + ) -> Option { + if !self.enabled(Feature::Goals) + || should_ignore_goal_for_mode(turn_context.collaboration_mode.mode) + { + return None; + } + let state_db = match self.state_db_for_thread_goals().await { + Ok(Some(state_db)) => state_db, + Ok(None) => return None, + Err(err) => { + tracing::warn!("failed to open state db at turn end: {err}"); + return None; + } + }; + match state_db + .thread_goals() + .get_thread_goal(self.conversation_id) + .await + { + Ok(Some(goal)) => Some(protocol_goal_status_from_state(goal.status)), + Ok(None) => None, + Err(err) => { + tracing::warn!("failed to read thread goal at turn end: {err}"); + None + } + } + } + pub(crate) async fn set_thread_goal( &self, turn_context: &TurnContext, diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index 5cb8f438ab..2cd4d5ff4f 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -8743,6 +8743,10 @@ async fn create_thread_goal_fills_empty_thread_preview() -> anyhow::Result<()> { Some("Keep improving the benchmark"), page.items[0].preview.as_deref() ); + assert_eq!( + Some(ThreadGoalStatus::Active), + sess.goal_status_at_turn_end(tc.as_ref()).await + ); Ok(()) } @@ -8823,6 +8827,10 @@ async fn budget_limited_accounting_steers_active_turn_without_aborting() -> anyh .expect("goal should remain persisted after accounting"); assert_eq!(codex_state::ThreadGoalStatus::BudgetLimited, goal.status); assert_eq!(25, goal.tokens_used); + assert_eq!( + Some(ThreadGoalStatus::BudgetLimited), + sess.goal_status_at_turn_end(tc.as_ref()).await + ); set_total_token_usage( &sess, diff --git a/codex-rs/core/src/tasks/mod.rs b/codex-rs/core/src/tasks/mod.rs index aa161c1274..6f075e190d 100644 --- a/codex-rs/core/src/tasks/mod.rs +++ b/codex-rs/core/src/tasks/mod.rs @@ -33,6 +33,7 @@ use crate::session::turn_context::TurnContext; use crate::state::ActiveTurn; use crate::state::RunningTask; use crate::state::TaskKind; +use codex_analytics::GoalStatusAtTurnEndFact; use codex_analytics::TurnTokenUsageFact; use codex_login::AuthManager; use codex_models_manager::manager::SharedModelsManager; @@ -515,14 +516,19 @@ impl Session { self.emit_turn_abort_lifecycle(reason.clone(), turn_context.extension_data.as_ref()) .await; } - if (aborted_turn || reason == TurnAbortReason::Interrupted) - && let Err(err) = self + if aborted_turn || reason == TurnAbortReason::Interrupted { + if let Err(err) = self .goal_runtime_apply(GoalRuntimeEvent::TaskAborted { turn_context: turn_context.as_deref(), }) .await - { - warn!("failed to apply goal runtime abort event: {err}"); + { + warn!("failed to apply goal runtime abort event: {err}"); + } + if let Some(turn_context) = turn_context.as_deref() { + self.track_goal_status_at_turn_end_analytics(turn_context) + .await; + } } if let Some(active_turn) = active_turn_to_clear { // Let interrupted tasks observe cancellation before dropping pending approvals, or an @@ -572,6 +578,10 @@ impl Session { { warn!("failed to apply goal runtime abort event: {err}"); } + if let Some(turn_context) = turn_context.as_deref() { + self.track_goal_status_at_turn_end_analytics(turn_context) + .await; + } // Let interrupted tasks observe cancellation before dropping pending approvals, or an // in-flight approval wait can surface as a model-visible rejection before TurnAborted. self.input_queue.clear_pending(&active_turn).await; @@ -774,6 +784,8 @@ impl Session { { warn!("failed to apply goal runtime turn-finished event: {err}"); } + self.track_goal_status_at_turn_end_analytics(turn_context.as_ref()) + .await; let event = EventMsg::TurnComplete(TurnCompleteEvent { turn_id: turn_context.sub_id.clone(), last_agent_message, @@ -811,6 +823,17 @@ impl Session { } } + async fn track_goal_status_at_turn_end_analytics(&self, turn_context: &TurnContext) { + let goal_status_at_turn_end = self.goal_status_at_turn_end(turn_context).await; + self.services + .analytics_events_client + .track_goal_status_at_turn_end(GoalStatusAtTurnEndFact { + turn_id: turn_context.sub_id.clone(), + thread_id: self.conversation_id.to_string(), + goal_status_at_turn_end, + }); + } + async fn take_active_turn(&self) -> Option { let mut active = self.active_turn.lock().await; active.take()