mirror of
https://github.com/openai/codex.git
synced 2026-10-21 17:48:44 +00:00
add goal_status_at_turn_start
This commit is contained in:
@@ -51,6 +51,7 @@ use crate::facts::CompactionStrategy;
|
||||
use crate::facts::CompactionTrigger;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::GoalStatusAtTurnEndFact;
|
||||
use crate::facts::GoalStatusAtTurnStartFact;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::InputError;
|
||||
@@ -345,6 +346,18 @@ fn sample_goal_status_at_turn_end_fact(
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_goal_status_at_turn_start_fact(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
goal_status_at_turn_start: Option<ThreadGoalStatus>,
|
||||
) -> GoalStatusAtTurnStartFact {
|
||||
GoalStatusAtTurnStartFact {
|
||||
thread_id: thread_id.to_string(),
|
||||
turn_id: turn_id.to_string(),
|
||||
goal_status_at_turn_start,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_turn_completed_notification(
|
||||
thread_id: &str,
|
||||
turn_id: &str,
|
||||
@@ -647,6 +660,14 @@ async fn ingest_turn_prerequisites(
|
||||
.await;
|
||||
}
|
||||
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(
|
||||
sample_goal_status_at_turn_start_fact("thread-2", "turn-2", None),
|
||||
))),
|
||||
out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
|
||||
@@ -3256,6 +3277,7 @@ fn turn_event_serializes_expected_shape() {
|
||||
approvals_reviewer: "auto_review".to_string(),
|
||||
sandbox_network_access: true,
|
||||
collaboration_mode: Some("plan"),
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
personality: Some("pragmatic".to_string()),
|
||||
num_input_images: 2,
|
||||
@@ -3319,6 +3341,7 @@ fn turn_event_serializes_expected_shape() {
|
||||
"approvals_reviewer": "auto_review",
|
||||
"sandbox_network_access": true,
|
||||
"collaboration_mode": "plan",
|
||||
"goal_status_at_turn_start": null,
|
||||
"goal_status_at_turn_end": null,
|
||||
"personality": "pragmatic",
|
||||
"num_input_images": 2,
|
||||
@@ -3596,6 +3619,18 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
/*include_token_usage*/ true,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(
|
||||
sample_goal_status_at_turn_start_fact(
|
||||
"thread-2",
|
||||
"turn-2",
|
||||
Some(ThreadGoalStatus::Active),
|
||||
),
|
||||
))),
|
||||
&mut out,
|
||||
)
|
||||
.await;
|
||||
reducer
|
||||
.ingest(
|
||||
AnalyticsFact::Custom(CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(
|
||||
@@ -3675,6 +3710,10 @@ async fn turn_lifecycle_emits_turn_event() {
|
||||
json!(13)
|
||||
);
|
||||
assert_eq!(payload["event_params"]["total_tokens"], json!(321));
|
||||
assert_eq!(
|
||||
payload["event_params"]["goal_status_at_turn_start"],
|
||||
json!("active")
|
||||
);
|
||||
assert_eq!(
|
||||
payload["event_params"]["goal_status_at_turn_end"],
|
||||
json!("budget_limited")
|
||||
|
||||
@@ -11,6 +11,7 @@ use crate::facts::AppMentionedInput;
|
||||
use crate::facts::AppUsedInput;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::GoalStatusAtTurnEndFact;
|
||||
use crate::facts::GoalStatusAtTurnStartFact;
|
||||
use crate::facts::HookRunFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::PluginState;
|
||||
@@ -257,6 +258,12 @@ impl AnalyticsEventsClient {
|
||||
)));
|
||||
}
|
||||
|
||||
pub fn track_goal_status_at_turn_start(&self, fact: GoalStatusAtTurnStartFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::GoalStatusAtTurnStart(Box::new(fact)),
|
||||
));
|
||||
}
|
||||
|
||||
pub fn track_goal_status_at_turn_end(&self, fact: GoalStatusAtTurnEndFact) {
|
||||
self.record_fact(AnalyticsFact::Custom(
|
||||
CustomAnalyticsFact::GoalStatusAtTurnEnd(Box::new(fact)),
|
||||
|
||||
@@ -792,6 +792,7 @@ pub(crate) struct CodexTurnEventParams {
|
||||
pub(crate) approvals_reviewer: String,
|
||||
pub(crate) sandbox_network_access: bool,
|
||||
pub(crate) collaboration_mode: Option<&'static str>,
|
||||
pub(crate) goal_status_at_turn_start: Option<&'static str>,
|
||||
pub(crate) goal_status_at_turn_end: Option<&'static str>,
|
||||
pub(crate) personality: Option<String>,
|
||||
pub(crate) num_input_images: usize,
|
||||
|
||||
@@ -100,6 +100,13 @@ pub struct TurnTokenUsageFact {
|
||||
pub token_usage: TokenUsage,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct GoalStatusAtTurnStartFact {
|
||||
pub turn_id: String,
|
||||
pub thread_id: String,
|
||||
pub goal_status_at_turn_start: Option<ThreadGoalStatus>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct GoalStatusAtTurnEndFact {
|
||||
pub turn_id: String,
|
||||
@@ -337,6 +344,7 @@ pub(crate) enum CustomAnalyticsFact {
|
||||
GuardianReview(Box<GuardianReviewEventParams>),
|
||||
TurnResolvedConfig(Box<TurnResolvedConfigFact>),
|
||||
TurnTokenUsage(Box<TurnTokenUsageFact>),
|
||||
GoalStatusAtTurnStart(Box<GoalStatusAtTurnStartFact>),
|
||||
GoalStatusAtTurnEnd(Box<GoalStatusAtTurnEndFact>),
|
||||
SkillInvoked(SkillInvokedInput),
|
||||
AppMentioned(AppMentionedInput),
|
||||
|
||||
@@ -32,6 +32,7 @@ pub use facts::CompactionStatus;
|
||||
pub use facts::CompactionStrategy;
|
||||
pub use facts::CompactionTrigger;
|
||||
pub use facts::GoalStatusAtTurnEndFact;
|
||||
pub use facts::GoalStatusAtTurnStartFact;
|
||||
pub use facts::HookRunFact;
|
||||
pub use facts::InputError;
|
||||
pub use facts::InvocationType;
|
||||
|
||||
@@ -65,6 +65,7 @@ use crate::facts::AppUsedInput;
|
||||
use crate::facts::CodexCompactionEvent;
|
||||
use crate::facts::CustomAnalyticsFact;
|
||||
use crate::facts::GoalStatusAtTurnEndFact;
|
||||
use crate::facts::GoalStatusAtTurnStartFact;
|
||||
use crate::facts::HookRunInput;
|
||||
use crate::facts::PluginState;
|
||||
use crate::facts::PluginStateChangedInput;
|
||||
@@ -326,7 +327,8 @@ struct TurnState {
|
||||
resolved_config: Option<TurnResolvedConfigFact>,
|
||||
started_at: Option<u64>,
|
||||
token_usage: Option<TokenUsage>,
|
||||
// Outer None means the completion fact has not arrived; inner None means no goal.
|
||||
// `None` means a boundary fact has not arrived; `Some(None)` means no applicable goal.
|
||||
goal_status_at_turn_start: Option<Option<ThreadGoalStatus>>,
|
||||
goal_status_at_turn_end: Option<Option<ThreadGoalStatus>>,
|
||||
completed: Option<CompletedTurnState>,
|
||||
latest_diff: Option<String>,
|
||||
@@ -468,6 +470,9 @@ impl AnalyticsReducer {
|
||||
CustomAnalyticsFact::TurnTokenUsage(input) => {
|
||||
self.ingest_turn_token_usage(*input, out).await;
|
||||
}
|
||||
CustomAnalyticsFact::GoalStatusAtTurnStart(input) => {
|
||||
self.ingest_goal_status_at_turn_start(*input, out).await;
|
||||
}
|
||||
CustomAnalyticsFact::GoalStatusAtTurnEnd(input) => {
|
||||
self.ingest_goal_status_at_turn_end(*input, out).await;
|
||||
}
|
||||
@@ -618,6 +623,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -643,6 +649,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -654,6 +661,31 @@ impl AnalyticsReducer {
|
||||
self.maybe_emit_turn_event(&turn_id, out).await;
|
||||
}
|
||||
|
||||
async fn ingest_goal_status_at_turn_start(
|
||||
&mut self,
|
||||
input: GoalStatusAtTurnStartFact,
|
||||
out: &mut Vec<TrackEventRequest>,
|
||||
) {
|
||||
let turn_id = input.turn_id.clone();
|
||||
let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState {
|
||||
connection_id: None,
|
||||
thread_id: None,
|
||||
num_input_images: None,
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
steer_count: 0,
|
||||
tool_counts: TurnToolCounts::default(),
|
||||
});
|
||||
turn_state.thread_id = Some(input.thread_id);
|
||||
turn_state.goal_status_at_turn_start = Some(input.goal_status_at_turn_start);
|
||||
self.maybe_emit_turn_event(&turn_id, out).await;
|
||||
}
|
||||
|
||||
async fn ingest_goal_status_at_turn_end(
|
||||
&mut self,
|
||||
input: GoalStatusAtTurnEndFact,
|
||||
@@ -667,6 +699,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -833,6 +866,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -1193,6 +1227,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -1215,6 +1250,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -1235,6 +1271,7 @@ impl AnalyticsReducer {
|
||||
resolved_config: None,
|
||||
started_at: None,
|
||||
token_usage: None,
|
||||
goal_status_at_turn_start: None,
|
||||
goal_status_at_turn_end: None,
|
||||
completed: None,
|
||||
latest_diff: None,
|
||||
@@ -1515,6 +1552,7 @@ impl AnalyticsReducer {
|
||||
if turn_state.thread_id.is_none()
|
||||
|| turn_state.num_input_images.is_none()
|
||||
|| turn_state.resolved_config.is_none()
|
||||
|| turn_state.goal_status_at_turn_start.is_none()
|
||||
|| turn_state.goal_status_at_turn_end.is_none()
|
||||
|| turn_state.completed.is_none()
|
||||
{
|
||||
@@ -2520,6 +2558,10 @@ fn codex_turn_event_params(
|
||||
approvals_reviewer: approvals_reviewer.to_string(),
|
||||
sandbox_network_access,
|
||||
collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)),
|
||||
goal_status_at_turn_start: turn_state
|
||||
.goal_status_at_turn_start
|
||||
.flatten()
|
||||
.map(goal_status_mode),
|
||||
goal_status_at_turn_end: turn_state
|
||||
.goal_status_at_turn_end
|
||||
.flatten()
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::state::TurnState;
|
||||
use crate::tasks::RegularTask;
|
||||
use crate::tools::handlers::goal_spec::UPDATE_GOAL_TOOL_NAME;
|
||||
use anyhow::Context;
|
||||
use codex_analytics::GoalStatusAtTurnStartFact;
|
||||
use codex_features::Feature;
|
||||
use codex_otel::GOAL_BLOCKED_METRIC;
|
||||
use codex_otel::GOAL_BUDGET_LIMITED_METRIC;
|
||||
@@ -347,8 +348,16 @@ impl Session {
|
||||
turn_context,
|
||||
token_usage,
|
||||
} => Box::pin(async move {
|
||||
self.mark_thread_goal_turn_started(turn_context, token_usage)
|
||||
let goal_status_at_turn_start = self
|
||||
.mark_thread_goal_turn_started(turn_context, token_usage)
|
||||
.await;
|
||||
self.services
|
||||
.analytics_events_client
|
||||
.track_goal_status_at_turn_start(GoalStatusAtTurnStartFact {
|
||||
turn_id: turn_context.sub_id.clone(),
|
||||
thread_id: self.conversation_id.to_string(),
|
||||
goal_status_at_turn_start,
|
||||
});
|
||||
Ok(())
|
||||
}),
|
||||
GoalRuntimeEvent::ToolCompleted {
|
||||
@@ -874,25 +883,25 @@ impl Session {
|
||||
&self,
|
||||
turn_context: &TurnContext,
|
||||
token_usage: TokenUsage,
|
||||
) {
|
||||
) -> Option<ThreadGoalStatus> {
|
||||
self.goal_runtime.accounting.lock().await.turn = Some(GoalTurnAccountingSnapshot::new(
|
||||
turn_context.sub_id.clone(),
|
||||
token_usage,
|
||||
));
|
||||
|
||||
if !self.enabled(Feature::Goals) {
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
if should_ignore_goal_for_mode(turn_context.collaboration_mode.mode) {
|
||||
self.clear_active_goal_accounting(turn_context).await;
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
let state_db = match self.state_db_for_thread_goals().await {
|
||||
Ok(Some(state_db)) => state_db,
|
||||
Ok(None) => return,
|
||||
Ok(None) => return None,
|
||||
Err(err) => {
|
||||
tracing::warn!("failed to open state db at turn start: {err}");
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
match state_db
|
||||
@@ -907,6 +916,7 @@ impl Session {
|
||||
| codex_state::ThreadGoalStatus::BudgetLimited
|
||||
) =>
|
||||
{
|
||||
let status = protocol_goal_status_from_state(goal.status);
|
||||
let mut accounting = self.goal_runtime.accounting.lock().await;
|
||||
if let Some(turn) = accounting.turn.as_mut()
|
||||
&& turn.turn_id == turn_context.sub_id
|
||||
@@ -914,17 +924,20 @@ impl Session {
|
||||
turn.mark_active_goal(goal.goal_id.clone());
|
||||
}
|
||||
accounting.wall_clock.mark_active_goal(goal.goal_id);
|
||||
Some(status)
|
||||
}
|
||||
Ok(Some(_)) | Ok(None) => {
|
||||
Ok(goal) => {
|
||||
self.goal_runtime
|
||||
.accounting
|
||||
.lock()
|
||||
.await
|
||||
.wall_clock
|
||||
.clear_active_goal();
|
||||
goal.map(|goal| protocol_goal_status_from_state(goal.status))
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!("failed to read thread goal at turn start: {err}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user