diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 2b43f88357..de1481c537 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1715,7 +1715,7 @@ async fn feature_observations_match_legacy_analytics_facts() { } #[tokio::test] -async fn turn_lifecycle_observations_match_legacy_sources() { +async fn turn_observations_match_legacy_sources() { let mut legacy_reducer = AnalyticsReducer::default(); let mut observation_reducer = AnalyticsObservationReducer::default(); let mut legacy_events = Vec::new(); @@ -1774,8 +1774,9 @@ async fn turn_lifecycle_observations_match_legacy_sources() { .await; observation_events.clear(); - // Keep not-yet-migrated lifecycle context identical on both sides. This - // test swaps terminal turn lifecycle and token accounting to observations. + // Keep not-yet-migrated request/thread context identical on both sides. + // This test swaps started/ended turn facts, including resolved config and + // token accounting, to observations while comparing the final payload. observation_reducer .ingest_existing_fact_for_test( AnalyticsFact::Request { @@ -1795,19 +1796,29 @@ async fn turn_lifecycle_observations_match_legacy_sources() { &mut observation_events, ) .await; - observation_reducer - .ingest_existing_fact_for_test( - AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( - sample_turn_resolved_config("turn-2"), - ))), - &mut observation_events, - ) - .await; observation_reducer .ingest_turn_started( codex_observability::events::TurnStarted { thread_id: "thread-2", turn_id: "turn-2", + config: codex_observability::events::TurnConfig { + num_input_images: 1, + submission_type: None, + ephemeral: false, + model: "gpt-5", + model_provider: "openai", + sandbox_mode: codex_observability::events::SandboxMode::ReadOnly, + sandbox_network_access: true, + reasoning_effort: None, + reasoning_summary: None, + service_tier: None, + approval_policy: codex_observability::events::ApprovalPolicy::OnRequest, + approval_reviewer: + codex_observability::events::ApprovalReviewer::GuardianSubagent, + collaboration_mode: codex_observability::events::CollaborationMode::Plan, + personality: None, + is_first_turn: true, + }, started_at: 455, }, &mut observation_events, diff --git a/codex-rs/analytics/src/observation_projection.rs b/codex-rs/analytics/src/observation_projection.rs index 20277e8bd9..77063bbfc2 100644 --- a/codex-rs/analytics/src/observation_projection.rs +++ b/codex-rs/analytics/src/observation_projection.rs @@ -17,6 +17,8 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnSubmissionType as AnalyticsTurnSubmissionType; use crate::facts::TurnTokenUsageFact; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::Turn; @@ -25,7 +27,19 @@ use codex_app_server_protocol::TurnStartedNotification; use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; use codex_login::default_client::originator; use codex_observability::events; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::HookRunStatus as ProtocolHookRunStatus; +use codex_protocol::protocol::NetworkAccess; +use codex_protocol::protocol::ReadOnlyAccess; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TokenUsage; pub(crate) fn app_mentioned_input(observation: events::AppMentioned<'_>) -> AppMentionedInput { @@ -143,6 +157,101 @@ pub(crate) fn turn_started_notification( }) } +pub(crate) fn turn_resolved_config_fact( + observation: &events::TurnStarted<'_>, +) -> TurnResolvedConfigFact { + let config = observation.config; + TurnResolvedConfigFact { + turn_id: observation.turn_id.to_string(), + thread_id: observation.thread_id.to_string(), + num_input_images: config.num_input_images, + submission_type: config + .submission_type + .map(|submission_type| match submission_type { + events::TurnSubmissionType::Default => AnalyticsTurnSubmissionType::Default, + events::TurnSubmissionType::Queued => AnalyticsTurnSubmissionType::Queued, + }), + ephemeral: config.ephemeral, + // The legacy fact carries session_source, but codex_turn_event derives + // thread source from thread lifecycle metadata instead. Keep this + // placeholder local to the projection until a consumer needs it. + session_source: SessionSource::Unknown, + model: config.model.to_string(), + model_provider: config.model_provider.to_string(), + sandbox_policy: match config.sandbox_mode { + events::SandboxMode::FullAccess => SandboxPolicy::DangerFullAccess, + events::SandboxMode::ReadOnly => SandboxPolicy::ReadOnly { + access: ReadOnlyAccess::FullAccess, + network_access: config.sandbox_network_access, + }, + events::SandboxMode::WorkspaceWrite => SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: ReadOnlyAccess::FullAccess, + network_access: config.sandbox_network_access, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }, + events::SandboxMode::ExternalSandbox => SandboxPolicy::ExternalSandbox { + network_access: if config.sandbox_network_access { + NetworkAccess::Enabled + } else { + NetworkAccess::Restricted + }, + }, + }, + reasoning_effort: config + .reasoning_effort + .map(|reasoning_effort| match reasoning_effort { + events::ReasoningEffort::None => ReasoningEffort::None, + events::ReasoningEffort::Minimal => ReasoningEffort::Minimal, + events::ReasoningEffort::Low => ReasoningEffort::Low, + events::ReasoningEffort::Medium => ReasoningEffort::Medium, + events::ReasoningEffort::High => ReasoningEffort::High, + events::ReasoningEffort::XHigh => ReasoningEffort::XHigh, + }), + reasoning_summary: config.reasoning_summary.map( + |reasoning_summary| match reasoning_summary { + events::ReasoningSummary::Auto => ReasoningSummary::Auto, + events::ReasoningSummary::Concise => ReasoningSummary::Concise, + events::ReasoningSummary::Detailed => ReasoningSummary::Detailed, + events::ReasoningSummary::None => ReasoningSummary::None, + }, + ), + service_tier: config.service_tier.map(|service_tier| match service_tier { + events::ServiceTier::Fast => ServiceTier::Fast, + events::ServiceTier::Flex => ServiceTier::Flex, + }), + approval_policy: match config.approval_policy { + events::ApprovalPolicy::Untrusted => AskForApproval::UnlessTrusted, + events::ApprovalPolicy::OnFailure => AskForApproval::OnFailure, + events::ApprovalPolicy::OnRequest => AskForApproval::OnRequest, + events::ApprovalPolicy::Granular => AskForApproval::Granular(GranularApprovalConfig { + sandbox_approval: true, + rules: true, + skill_approval: true, + request_permissions: true, + mcp_elicitations: true, + }), + events::ApprovalPolicy::Never => AskForApproval::Never, + }, + approvals_reviewer: match config.approval_reviewer { + events::ApprovalReviewer::User => ApprovalsReviewer::User, + events::ApprovalReviewer::GuardianSubagent => ApprovalsReviewer::GuardianSubagent, + }, + sandbox_network_access: config.sandbox_network_access, + collaboration_mode: match config.collaboration_mode { + events::CollaborationMode::Default => ModeKind::Default, + events::CollaborationMode::Plan => ModeKind::Plan, + }, + personality: config.personality.map(|personality| match personality { + events::Personality::None => Personality::None, + events::Personality::Friendly => Personality::Friendly, + events::Personality::Pragmatic => Personality::Pragmatic, + }), + is_first_turn: config.is_first_turn, + } +} + pub(crate) fn turn_token_usage_fact( observation: &events::TurnEnded<'_>, ) -> Option { diff --git a/codex-rs/analytics/src/observation_reducer.rs b/codex-rs/analytics/src/observation_reducer.rs index 4907911551..d40eaecaa3 100644 --- a/codex-rs/analytics/src/observation_reducer.rs +++ b/codex-rs/analytics/src/observation_reducer.rs @@ -72,6 +72,15 @@ impl AnalyticsObservationReducer { observation: events::TurnStarted<'_>, out: &mut Vec, ) { + self.legacy + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + observation_projection::turn_resolved_config_fact(&observation), + ))), + out, + ) + .await; + self.legacy .ingest( AnalyticsFact::Notification(Box::new( diff --git a/codex-rs/observability/src/events.rs b/codex-rs/observability/src/events.rs index 5c15b99cba..444696f330 100644 --- a/codex-rs/observability/src/events.rs +++ b/codex-rs/observability/src/events.rs @@ -3,6 +3,10 @@ use crate::Observation; use serde::Serialize; +mod turn; + +pub use turn::*; + /// How an app/tool/plugin capability was selected by the user or system. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] @@ -41,87 +45,6 @@ pub enum PluginState { Disabled, } -/// Terminal turn status after Codex stops working on a turn. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum TurnStatus { - /// The turn completed successfully. - Completed, - /// The turn failed with an error. - Failed, - /// The turn was interrupted before normal completion. - Interrupted, -} - -/// Observation emitted when execution of a turn starts. -#[derive(Observation)] -#[observation(name = "turn.started", crate = "crate", uses = ["analytics"])] -pub struct TurnStarted<'a> { - /// Thread that owns the turn. - #[obs(level = "basic", class = "identifier")] - pub thread_id: &'a str, - - /// Turn that started. - #[obs(level = "basic", class = "identifier")] - pub turn_id: &'a str, - - /// Unix timestamp in seconds when the turn started. - #[obs(level = "basic", class = "operational")] - pub started_at: i64, -} - -/// Token accounting reported for a completed turn. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] -pub struct TurnTokenUsage { - /// Prompt/input token count reported by the model provider. - pub input_tokens: i64, - - /// Input tokens served from provider-side cache. - pub cached_input_tokens: i64, - - /// Output token count reported by the model provider. - pub output_tokens: i64, - - /// Output tokens spent on model reasoning. - pub reasoning_output_tokens: i64, - - /// Total token count reported by the model provider. - pub total_tokens: i64, -} - -/// Observation emitted when a turn reaches a terminal state. -#[derive(Observation)] -#[observation(name = "turn.ended", crate = "crate", uses = ["analytics"])] -pub struct TurnEnded<'a> { - /// Thread that owns the turn. - #[obs(level = "basic", class = "identifier")] - pub thread_id: &'a str, - - /// Turn that reached a terminal state. - #[obs(level = "basic", class = "identifier")] - pub turn_id: &'a str, - - /// Terminal status for the turn. - #[obs(level = "basic", class = "operational")] - pub status: TurnStatus, - - /// Token usage reported by the model provider. - /// - /// This is absent when a turn ends before provider usage is available, for - /// example an early local failure, an interruption, or a provider failure - /// before a usage-bearing response is received. - #[obs(level = "basic", class = "operational")] - pub token_usage: Option, - - /// Unix timestamp in seconds when the turn ended. - #[obs(level = "basic", class = "operational")] - pub ended_at: i64, - - /// Turn duration in milliseconds. - #[obs(level = "basic", class = "operational")] - pub duration_ms: i64, -} - /// Observation emitted when an app connector is mentioned during a turn. #[derive(Observation)] #[observation(name = "app.mentioned", crate = "crate", uses = ["analytics"])] diff --git a/codex-rs/observability/src/events/turn.rs b/codex-rs/observability/src/events/turn.rs new file mode 100644 index 0000000000..842973e08b --- /dev/null +++ b/codex-rs/observability/src/events/turn.rs @@ -0,0 +1,178 @@ +//! Turn lifecycle observation event definitions. + +use crate::Observation; +use serde::Serialize; + +/// Terminal turn status after Codex stops working on a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, +} + +/// How a turn was submitted for execution. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +/// Filesystem sandbox mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SandboxMode { + FullAccess, + ReadOnly, + WorkspaceWrite, + ExternalSandbox, +} + +/// Approval policy resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ApprovalPolicy { + /// Legacy name for the policy that asks unless an action is trusted. + Untrusted, + OnFailure, + OnRequest, + Granular, + Never, +} + +/// Destination that reviews approval requests for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ApprovalReviewer { + User, + GuardianSubagent, +} + +/// Collaboration mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollaborationMode { + Default, + Plan, +} + +/// Reasoning effort resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, + XHigh, +} + +/// Reasoning summary mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningSummary { + Auto, + Concise, + Detailed, + None, +} + +/// Service tier resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ServiceTier { + Fast, + Flex, +} + +/// Response personality resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Personality { + None, + Friendly, + Pragmatic, +} + +/// Configuration resolved before a turn starts executing. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct TurnConfig<'a> { + pub num_input_images: usize, + /// Absent when the caller cannot distinguish default from queued submission. + pub submission_type: Option, + pub ephemeral: bool, + pub model: &'a str, + pub model_provider: &'a str, + pub sandbox_mode: SandboxMode, + /// Kept separate from sandbox mode because analytics reports network + /// capability as an independent resolved setting. + pub sandbox_network_access: bool, + /// Absent when the selected model/provider has no explicit effort setting. + pub reasoning_effort: Option, + /// None means no summary setting was resolved; Some(None) means summaries + /// were explicitly disabled. + pub reasoning_summary: Option, + pub service_tier: Option, + pub approval_policy: ApprovalPolicy, + pub approval_reviewer: ApprovalReviewer, + pub collaboration_mode: CollaborationMode, + /// Absent when no personality setting was resolved. + pub personality: Option, + pub is_first_turn: bool, +} + +/// Observation emitted when execution of a turn starts. +#[derive(Observation)] +#[observation(name = "turn.started", crate = "crate", uses = ["analytics"])] +pub struct TurnStarted<'a> { + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + pub config: TurnConfig<'a>, + + /// Unix timestamp in seconds when the turn started. + #[obs(level = "basic", class = "operational")] + pub started_at: i64, +} + +/// Token accounting reported for a completed turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct TurnTokenUsage { + pub input_tokens: i64, + pub cached_input_tokens: i64, + pub output_tokens: i64, + pub reasoning_output_tokens: i64, + pub total_tokens: i64, +} + +/// Observation emitted when a turn reaches a terminal state. +#[derive(Observation)] +#[observation(name = "turn.ended", crate = "crate", uses = ["analytics"])] +pub struct TurnEnded<'a> { + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + pub status: TurnStatus, + + /// Absent when a turn ends before provider usage is available. + #[obs(level = "basic", class = "operational")] + pub token_usage: Option, + + /// Unix timestamp in seconds when the turn ended. + #[obs(level = "basic", class = "operational")] + pub ended_at: i64, + + #[obs(level = "basic", class = "operational")] + pub duration_ms: i64, +} diff --git a/docs/observability-event-stream-design.md b/docs/observability-event-stream-design.md index 9a8e3969f3..2ca1bfd5b1 100644 --- a/docs/observability-event-stream-design.md +++ b/docs/observability-event-stream-design.md @@ -145,8 +145,8 @@ Example: use codex_observability::Observation; #[derive(Observation)] -#[observation(name = "turn.config_resolved", uses = ["analytics"])] -struct TurnConfigResolved<'a> { +#[observation(name = "turn.started", uses = ["analytics"])] +struct TurnStarted<'a> { #[obs(level = "basic", class = "identifier")] thread_id: &'a str, @@ -154,7 +154,7 @@ struct TurnConfigResolved<'a> { turn_id: &'a str, #[obs(level = "basic", class = "operational")] - model: &'a str, + started_at: i64, } ``` @@ -292,7 +292,7 @@ output schemas. | Observations | Analytics output | | --- | --- | | `thread.started` | `codex_thread_initialized` | -| `turn.started`, `turn.ended` with token usage, and future config observations | `codex_turn_event` | +| `turn.started` with resolved config, `turn.ended` with token usage | `codex_turn_event` | | `turn.steer_resolved` | `codex_turn_steer_event` | | compaction lifecycle observations | `codex_compaction_event` | | skill/app/plugin/guardian observations | existing feature events |