add goal_status_at_turn_start

This commit is contained in:
Mark Steinbrick
2026-05-27 13:41:00 -07:00
parent a339192d1b
commit bb1770753d
7 changed files with 119 additions and 8 deletions
@@ -51,6 +51,7 @@ use crate::facts::CompactionStrategy;
use crate::facts::CompactionTrigger;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::GoalStatusAtTurnStartFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::InputError;
@@ -345,6 +346,18 @@ fn sample_goal_status_at_turn_end_fact(
}
}
fn sample_goal_status_at_turn_start_fact(
thread_id: &str,
turn_id: &str,
goal_status_at_turn_start: Option<ThreadGoalStatus>,
) -> GoalStatusAtTurnStartFact {
GoalStatusAtTurnStartFact {
thread_id: thread_id.to_string(),
turn_id: turn_id.to_string(),
goal_status_at_turn_start,
}
}
fn sample_turn_completed_notification(
thread_id: &str,
turn_id: &str,
@@ -647,6 +660,14 @@ async fn ingest_turn_prerequisites(
.await;
}
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(
sample_goal_status_at_turn_start_fact("thread-2", "turn-2", None),
))),
out,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
@@ -3256,6 +3277,7 @@ fn turn_event_serializes_expected_shape() {
approvals_reviewer: "auto_review".to_string(),
sandbox_network_access: true,
collaboration_mode: Some("plan"),
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
personality: Some("pragmatic".to_string()),
num_input_images: 2,
@@ -3319,6 +3341,7 @@ fn turn_event_serializes_expected_shape() {
"approvals_reviewer": "auto_review",
"sandbox_network_access": true,
"collaboration_mode": "plan",
"goal_status_at_turn_start": null,
"goal_status_at_turn_end": null,
"personality": "pragmatic",
"num_input_images": 2,
@@ -3596,6 +3619,18 @@ async fn turn_lifecycle_emits_turn_event() {
/*include_token_usage*/ true,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(
sample_goal_status_at_turn_start_fact(
"thread-2",
"turn-2",
Some(ThreadGoalStatus::Active),
),
))),
&mut out,
)
.await;
reducer
.ingest(
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
@@ -3675,6 +3710,10 @@ async fn turn_lifecycle_emits_turn_event() {
json!(13)
);
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
assert_eq!(
payload["event_params"]["goal_status_at_turn_start"],
json!("active")
);
assert_eq!(
payload["event_params"]["goal_status_at_turn_end"],
json!("budget_limited")
+7
View File
@@ -11,6 +11,7 @@ use crate::facts::AppMentionedInput;
use crate::facts::AppUsedInput;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::GoalStatusAtTurnStartFact;
use crate::facts::HookRunFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
@@ -257,6 +258,12 @@ impl AnalyticsEventsClient {
)));
}
pub fn track_goal_status_at_turn_start(&self, fact: GoalStatusAtTurnStartFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(fact)),
));
}
pub fn track_goal_status_at_turn_end(&self, fact: GoalStatusAtTurnEndFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(fact)),
+1
View File
@@ -792,6 +792,7 @@ pub(crate) struct CodexTurnEventParams {
pub(crate) approvals_reviewer: String,
pub(crate) sandbox_network_access: bool,
pub(crate) collaboration_mode: Option<&'static str>,
pub(crate) goal_status_at_turn_start: Option<&'static str>,
pub(crate) goal_status_at_turn_end: Option<&'static str>,
pub(crate) personality: Option<String>,
pub(crate) num_input_images: usize,
+8
View File
@@ -100,6 +100,13 @@ pub struct TurnTokenUsageFact {
pub token_usage: TokenUsage,
}
#[derive(Clone)]
pub struct GoalStatusAtTurnStartFact {
pub turn_id: String,
pub thread_id: String,
pub goal_status_at_turn_start: Option<ThreadGoalStatus>,
}
#[derive(Clone)]
pub struct GoalStatusAtTurnEndFact {
pub turn_id: String,
@@ -337,6 +344,7 @@ pub(crate) enum CustomAnalyticsFact {
GuardianReview(Box<GuardianReviewEventParams>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
TurnTokenUsage(Box<TurnTokenUsageFact>),
GoalStatusAtTurnStart(Box<GoalStatusAtTurnStartFact>),
GoalStatusAtTurnEnd(Box<GoalStatusAtTurnEndFact>),
SkillInvoked(SkillInvokedInput),
AppMentioned(AppMentionedInput),
+1
View File
@@ -32,6 +32,7 @@ pub use facts::CompactionStatus;
pub use facts::CompactionStrategy;
pub use facts::CompactionTrigger;
pub use facts::GoalStatusAtTurnEndFact;
pub use facts::GoalStatusAtTurnStartFact;
pub use facts::HookRunFact;
pub use facts::InputError;
pub use facts::InvocationType;
+43 -1
View File
@@ -65,6 +65,7 @@ use crate::facts::AppUsedInput;
use crate::facts::CodexCompactionEvent;
use crate::facts::CustomAnalyticsFact;
use crate::facts::GoalStatusAtTurnEndFact;
use crate::facts::GoalStatusAtTurnStartFact;
use crate::facts::HookRunInput;
use crate::facts::PluginState;
use crate::facts::PluginStateChangedInput;
@@ -326,7 +327,8 @@ struct TurnState {
resolved_config: Option<TurnResolvedConfigFact>,
started_at: Option<u64>,
token_usage: Option<TokenUsage>,
// Outer None means the completion fact has not arrived; inner None means no goal.
// `None` means a boundary fact has not arrived; `Some(None)` means no applicable goal.
goal_status_at_turn_start: Option<Option<ThreadGoalStatus>>,
goal_status_at_turn_end: Option<Option<ThreadGoalStatus>>,
completed: Option<CompletedTurnState>,
latest_diff: Option<String>,
@@ -468,6 +470,9 @@ impl AnalyticsReducer {
CustomAnalyticsFact::TurnTokenUsage(input) => {
self.ingest_turn_token_usage(*input, out).await;
}
CustomAnalyticsFact::GoalStatusAtTurnStart(input) => {
self.ingest_goal_status_at_turn_start(*input, out).await;
}
CustomAnalyticsFact::GoalStatusAtTurnEnd(input) => {
self.ingest_goal_status_at_turn_end(*input, out).await;
}
@@ -618,6 +623,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -643,6 +649,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -654,6 +661,31 @@ impl AnalyticsReducer {
self.maybe_emit_turn_event(&turn_id, out).await;
}
async fn ingest_goal_status_at_turn_start(
&mut self,
input: GoalStatusAtTurnStartFact,
out: &mut Vec<TrackEventRequest>,
) {
let turn_id = input.turn_id.clone();
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
connection_id: None,
thread_id: None,
num_input_images: None,
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
steer_count: 0,
tool_counts: TurnToolCounts::default(),
});
turn_state.thread_id = Some(input.thread_id);
turn_state.goal_status_at_turn_start = Some(input.goal_status_at_turn_start);
self.maybe_emit_turn_event(&turn_id, out).await;
}
async fn ingest_goal_status_at_turn_end(
&mut self,
input: GoalStatusAtTurnEndFact,
@@ -667,6 +699,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -833,6 +866,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -1193,6 +1227,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -1215,6 +1250,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -1235,6 +1271,7 @@ impl AnalyticsReducer {
resolved_config: None,
started_at: None,
token_usage: None,
goal_status_at_turn_start: None,
goal_status_at_turn_end: None,
completed: None,
latest_diff: None,
@@ -1515,6 +1552,7 @@ impl AnalyticsReducer {
if turn_state.thread_id.is_none()
|| turn_state.num_input_images.is_none()
|| turn_state.resolved_config.is_none()
|| turn_state.goal_status_at_turn_start.is_none()
|| turn_state.goal_status_at_turn_end.is_none()
|| turn_state.completed.is_none()
{
@@ -2520,6 +2558,10 @@ fn codex_turn_event_params(
approvals_reviewer: approvals_reviewer.to_string(),
sandbox_network_access,
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
goal_status_at_turn_start: turn_state
.goal_status_at_turn_start
.flatten()
.map(goal_status_mode),
goal_status_at_turn_end: turn_state
.goal_status_at_turn_end
.flatten()
+20 -7
View File
@@ -15,6 +15,7 @@ use crate::state::TurnState;
use crate::tasks::RegularTask;
use crate::tools::handlers::goal_spec::UPDATE_GOAL_TOOL_NAME;
use anyhow::Context;
use codex_analytics::GoalStatusAtTurnStartFact;
use codex_features::Feature;
use codex_otel::GOAL_BLOCKED_METRIC;
use codex_otel::GOAL_BUDGET_LIMITED_METRIC;
@@ -347,8 +348,16 @@ impl Session {
turn_context,
token_usage,
} => Box::pin(async move {
self.mark_thread_goal_turn_started(turn_context, token_usage)
let goal_status_at_turn_start = self
.mark_thread_goal_turn_started(turn_context, token_usage)
.await;
self.services
.analytics_events_client
.track_goal_status_at_turn_start(GoalStatusAtTurnStartFact {
turn_id: turn_context.sub_id.clone(),
thread_id: self.conversation_id.to_string(),
goal_status_at_turn_start,
});
Ok(())
}),
GoalRuntimeEvent::ToolCompleted {
@@ -874,25 +883,25 @@ impl Session {
&self,
turn_context: &TurnContext,
token_usage: TokenUsage,
) {
) -> Option<ThreadGoalStatus> {
self.goal_runtime.accounting.lock().await.turn = Some(GoalTurnAccountingSnapshot::new(
turn_context.sub_id.clone(),
token_usage,
));
if !self.enabled(Feature::Goals) {
return;
return None;
}
if should_ignore_goal_for_mode(turn_context.collaboration_mode.mode) {
self.clear_active_goal_accounting(turn_context).await;
return;
return None;
}
let state_db = match self.state_db_for_thread_goals().await {
Ok(Some(state_db)) => state_db,
Ok(None) => return,
Ok(None) => return None,
Err(err) => {
tracing::warn!("failed to open state db at turn start: {err}");
return;
return None;
}
};
match state_db
@@ -907,6 +916,7 @@ impl Session {
| codex_state::ThreadGoalStatus::BudgetLimited
) =>
{
let status = protocol_goal_status_from_state(goal.status);
let mut accounting = self.goal_runtime.accounting.lock().await;
if let Some(turn) = accounting.turn.as_mut()
&& turn.turn_id == turn_context.sub_id
@@ -914,17 +924,20 @@ impl Session {
turn.mark_active_goal(goal.goal_id.clone());
}
accounting.wall_clock.mark_active_goal(goal.goal_id);
Some(status)
}
Ok(Some(_)) | Ok(None) => {
Ok(goal) => {
self.goal_runtime
.accounting
.lock()
.await
.wall_clock
.clear_active_goal();
goal.map(|goal| protocol_goal_status_from_state(goal.status))
}
Err(err) => {
tracing::warn!("failed to read thread goal at turn start: {err}");
None
}
}
}