From 3198988a52e39aa2568dfd110921d048cd4f0ba2 Mon Sep 17 00:00:00 2001 From: Albin Cassirer Date: Sun, 19 Apr 2026 16:08:18 -0700 Subject: [PATCH] [observability] Fold turn config into turn started\n\nMove turn-specific observation definitions into an events::turn module and\nre-export them from the shared event namespace.\n\nExtend turn.started with resolved configuration so the canonical event models\nthe runtime fact directly instead of mirroring the analytics reducer's separate\nconfig fact. Project turn.started into the existing resolved-config fact plus\nturn-start notification, and update conformance coverage so the observation path\nmatches the legacy codex_turn_event output.\n\nUpdate the design doc to describe resolved turn config as part of turn.started. --- .../analytics/src/analytics_client_tests.rs | 33 ++-- .../analytics/src/observation_projection.rs | 109 +++++++++++ codex-rs/analytics/src/observation_reducer.rs | 9 + codex-rs/observability/src/events.rs | 85 +-------- codex-rs/observability/src/events/turn.rs | 178 ++++++++++++++++++ docs/observability-event-stream-design.md | 8 +- 6 files changed, 326 insertions(+), 96 deletions(-) create mode 100644 codex-rs/observability/src/events/turn.rs diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 2b43f88357..de1481c537 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -1715,7 +1715,7 @@ async fn feature_observations_match_legacy_analytics_facts() { } #[tokio::test] -async fn turn_lifecycle_observations_match_legacy_sources() { +async fn turn_observations_match_legacy_sources() { let mut legacy_reducer = AnalyticsReducer::default(); let mut observation_reducer = AnalyticsObservationReducer::default(); let mut legacy_events = Vec::new(); @@ -1774,8 +1774,9 @@ async fn turn_lifecycle_observations_match_legacy_sources() { .await; observation_events.clear(); - // Keep not-yet-migrated lifecycle context identical on both sides. This - // test swaps terminal turn lifecycle and token accounting to observations. + // Keep not-yet-migrated request/thread context identical on both sides. + // This test swaps started/ended turn facts, including resolved config and + // token accounting, to observations while comparing the final payload. observation_reducer .ingest_existing_fact_for_test( AnalyticsFact::Request { @@ -1795,19 +1796,29 @@ async fn turn_lifecycle_observations_match_legacy_sources() { &mut observation_events, ) .await; - observation_reducer - .ingest_existing_fact_for_test( - AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( - sample_turn_resolved_config("turn-2"), - ))), - &mut observation_events, - ) - .await; observation_reducer .ingest_turn_started( codex_observability::events::TurnStarted { thread_id: "thread-2", turn_id: "turn-2", + config: codex_observability::events::TurnConfig { + num_input_images: 1, + submission_type: None, + ephemeral: false, + model: "gpt-5", + model_provider: "openai", + sandbox_mode: codex_observability::events::SandboxMode::ReadOnly, + sandbox_network_access: true, + reasoning_effort: None, + reasoning_summary: None, + service_tier: None, + approval_policy: codex_observability::events::ApprovalPolicy::OnRequest, + approval_reviewer: + codex_observability::events::ApprovalReviewer::GuardianSubagent, + collaboration_mode: codex_observability::events::CollaborationMode::Plan, + personality: None, + is_first_turn: true, + }, started_at: 455, }, &mut observation_events, diff --git a/codex-rs/analytics/src/observation_projection.rs b/codex-rs/analytics/src/observation_projection.rs index 20277e8bd9..77063bbfc2 100644 --- a/codex-rs/analytics/src/observation_projection.rs +++ b/codex-rs/analytics/src/observation_projection.rs @@ -17,6 +17,8 @@ use crate::facts::AppInvocation; use crate::facts::AppMentionedInput; use crate::facts::AppUsedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnSubmissionType as AnalyticsTurnSubmissionType; use crate::facts::TurnTokenUsageFact; use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::Turn; @@ -25,7 +27,19 @@ use codex_app_server_protocol::TurnStartedNotification; use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; use codex_login::default_client::originator; use codex_observability::events; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::GranularApprovalConfig; use codex_protocol::protocol::HookRunStatus as ProtocolHookRunStatus; +use codex_protocol::protocol::NetworkAccess; +use codex_protocol::protocol::ReadOnlyAccess; +use codex_protocol::protocol::SandboxPolicy; +use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::TokenUsage; pub(crate) fn app_mentioned_input(observation: events::AppMentioned<'_>) -> AppMentionedInput { @@ -143,6 +157,101 @@ pub(crate) fn turn_started_notification( }) } +pub(crate) fn turn_resolved_config_fact( + observation: &events::TurnStarted<'_>, +) -> TurnResolvedConfigFact { + let config = observation.config; + TurnResolvedConfigFact { + turn_id: observation.turn_id.to_string(), + thread_id: observation.thread_id.to_string(), + num_input_images: config.num_input_images, + submission_type: config + .submission_type + .map(|submission_type| match submission_type { + events::TurnSubmissionType::Default => AnalyticsTurnSubmissionType::Default, + events::TurnSubmissionType::Queued => AnalyticsTurnSubmissionType::Queued, + }), + ephemeral: config.ephemeral, + // The legacy fact carries session_source, but codex_turn_event derives + // thread source from thread lifecycle metadata instead. Keep this + // placeholder local to the projection until a consumer needs it. + session_source: SessionSource::Unknown, + model: config.model.to_string(), + model_provider: config.model_provider.to_string(), + sandbox_policy: match config.sandbox_mode { + events::SandboxMode::FullAccess => SandboxPolicy::DangerFullAccess, + events::SandboxMode::ReadOnly => SandboxPolicy::ReadOnly { + access: ReadOnlyAccess::FullAccess, + network_access: config.sandbox_network_access, + }, + events::SandboxMode::WorkspaceWrite => SandboxPolicy::WorkspaceWrite { + writable_roots: Vec::new(), + read_only_access: ReadOnlyAccess::FullAccess, + network_access: config.sandbox_network_access, + exclude_tmpdir_env_var: false, + exclude_slash_tmp: false, + }, + events::SandboxMode::ExternalSandbox => SandboxPolicy::ExternalSandbox { + network_access: if config.sandbox_network_access { + NetworkAccess::Enabled + } else { + NetworkAccess::Restricted + }, + }, + }, + reasoning_effort: config + .reasoning_effort + .map(|reasoning_effort| match reasoning_effort { + events::ReasoningEffort::None => ReasoningEffort::None, + events::ReasoningEffort::Minimal => ReasoningEffort::Minimal, + events::ReasoningEffort::Low => ReasoningEffort::Low, + events::ReasoningEffort::Medium => ReasoningEffort::Medium, + events::ReasoningEffort::High => ReasoningEffort::High, + events::ReasoningEffort::XHigh => ReasoningEffort::XHigh, + }), + reasoning_summary: config.reasoning_summary.map( + |reasoning_summary| match reasoning_summary { + events::ReasoningSummary::Auto => ReasoningSummary::Auto, + events::ReasoningSummary::Concise => ReasoningSummary::Concise, + events::ReasoningSummary::Detailed => ReasoningSummary::Detailed, + events::ReasoningSummary::None => ReasoningSummary::None, + }, + ), + service_tier: config.service_tier.map(|service_tier| match service_tier { + events::ServiceTier::Fast => ServiceTier::Fast, + events::ServiceTier::Flex => ServiceTier::Flex, + }), + approval_policy: match config.approval_policy { + events::ApprovalPolicy::Untrusted => AskForApproval::UnlessTrusted, + events::ApprovalPolicy::OnFailure => AskForApproval::OnFailure, + events::ApprovalPolicy::OnRequest => AskForApproval::OnRequest, + events::ApprovalPolicy::Granular => AskForApproval::Granular(GranularApprovalConfig { + sandbox_approval: true, + rules: true, + skill_approval: true, + request_permissions: true, + mcp_elicitations: true, + }), + events::ApprovalPolicy::Never => AskForApproval::Never, + }, + approvals_reviewer: match config.approval_reviewer { + events::ApprovalReviewer::User => ApprovalsReviewer::User, + events::ApprovalReviewer::GuardianSubagent => ApprovalsReviewer::GuardianSubagent, + }, + sandbox_network_access: config.sandbox_network_access, + collaboration_mode: match config.collaboration_mode { + events::CollaborationMode::Default => ModeKind::Default, + events::CollaborationMode::Plan => ModeKind::Plan, + }, + personality: config.personality.map(|personality| match personality { + events::Personality::None => Personality::None, + events::Personality::Friendly => Personality::Friendly, + events::Personality::Pragmatic => Personality::Pragmatic, + }), + is_first_turn: config.is_first_turn, + } +} + pub(crate) fn turn_token_usage_fact( observation: &events::TurnEnded<'_>, ) -> Option { diff --git a/codex-rs/analytics/src/observation_reducer.rs b/codex-rs/analytics/src/observation_reducer.rs index 4907911551..d40eaecaa3 100644 --- a/codex-rs/analytics/src/observation_reducer.rs +++ b/codex-rs/analytics/src/observation_reducer.rs @@ -72,6 +72,15 @@ impl AnalyticsObservationReducer { observation: events::TurnStarted<'_>, out: &mut Vec, ) { + self.legacy + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + observation_projection::turn_resolved_config_fact(&observation), + ))), + out, + ) + .await; + self.legacy .ingest( AnalyticsFact::Notification(Box::new( diff --git a/codex-rs/observability/src/events.rs b/codex-rs/observability/src/events.rs index 5c15b99cba..444696f330 100644 --- a/codex-rs/observability/src/events.rs +++ b/codex-rs/observability/src/events.rs @@ -3,6 +3,10 @@ use crate::Observation; use serde::Serialize; +mod turn; + +pub use turn::*; + /// How an app/tool/plugin capability was selected by the user or system. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] @@ -41,87 +45,6 @@ pub enum PluginState { Disabled, } -/// Terminal turn status after Codex stops working on a turn. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] -#[serde(rename_all = "snake_case")] -pub enum TurnStatus { - /// The turn completed successfully. - Completed, - /// The turn failed with an error. - Failed, - /// The turn was interrupted before normal completion. - Interrupted, -} - -/// Observation emitted when execution of a turn starts. -#[derive(Observation)] -#[observation(name = "turn.started", crate = "crate", uses = ["analytics"])] -pub struct TurnStarted<'a> { - /// Thread that owns the turn. - #[obs(level = "basic", class = "identifier")] - pub thread_id: &'a str, - - /// Turn that started. - #[obs(level = "basic", class = "identifier")] - pub turn_id: &'a str, - - /// Unix timestamp in seconds when the turn started. - #[obs(level = "basic", class = "operational")] - pub started_at: i64, -} - -/// Token accounting reported for a completed turn. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] -pub struct TurnTokenUsage { - /// Prompt/input token count reported by the model provider. - pub input_tokens: i64, - - /// Input tokens served from provider-side cache. - pub cached_input_tokens: i64, - - /// Output token count reported by the model provider. - pub output_tokens: i64, - - /// Output tokens spent on model reasoning. - pub reasoning_output_tokens: i64, - - /// Total token count reported by the model provider. - pub total_tokens: i64, -} - -/// Observation emitted when a turn reaches a terminal state. -#[derive(Observation)] -#[observation(name = "turn.ended", crate = "crate", uses = ["analytics"])] -pub struct TurnEnded<'a> { - /// Thread that owns the turn. - #[obs(level = "basic", class = "identifier")] - pub thread_id: &'a str, - - /// Turn that reached a terminal state. - #[obs(level = "basic", class = "identifier")] - pub turn_id: &'a str, - - /// Terminal status for the turn. - #[obs(level = "basic", class = "operational")] - pub status: TurnStatus, - - /// Token usage reported by the model provider. - /// - /// This is absent when a turn ends before provider usage is available, for - /// example an early local failure, an interruption, or a provider failure - /// before a usage-bearing response is received. - #[obs(level = "basic", class = "operational")] - pub token_usage: Option, - - /// Unix timestamp in seconds when the turn ended. - #[obs(level = "basic", class = "operational")] - pub ended_at: i64, - - /// Turn duration in milliseconds. - #[obs(level = "basic", class = "operational")] - pub duration_ms: i64, -} - /// Observation emitted when an app connector is mentioned during a turn. #[derive(Observation)] #[observation(name = "app.mentioned", crate = "crate", uses = ["analytics"])] diff --git a/codex-rs/observability/src/events/turn.rs b/codex-rs/observability/src/events/turn.rs new file mode 100644 index 0000000000..842973e08b --- /dev/null +++ b/codex-rs/observability/src/events/turn.rs @@ -0,0 +1,178 @@ +//! Turn lifecycle observation event definitions. + +use crate::Observation; +use serde::Serialize; + +/// Terminal turn status after Codex stops working on a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, +} + +/// How a turn was submitted for execution. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +/// Filesystem sandbox mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SandboxMode { + FullAccess, + ReadOnly, + WorkspaceWrite, + ExternalSandbox, +} + +/// Approval policy resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ApprovalPolicy { + /// Legacy name for the policy that asks unless an action is trusted. + Untrusted, + OnFailure, + OnRequest, + Granular, + Never, +} + +/// Destination that reviews approval requests for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ApprovalReviewer { + User, + GuardianSubagent, +} + +/// Collaboration mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CollaborationMode { + Default, + Plan, +} + +/// Reasoning effort resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, + XHigh, +} + +/// Reasoning summary mode resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningSummary { + Auto, + Concise, + Detailed, + None, +} + +/// Service tier resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum ServiceTier { + Fast, + Flex, +} + +/// Response personality resolved for a turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Personality { + None, + Friendly, + Pragmatic, +} + +/// Configuration resolved before a turn starts executing. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct TurnConfig<'a> { + pub num_input_images: usize, + /// Absent when the caller cannot distinguish default from queued submission. + pub submission_type: Option, + pub ephemeral: bool, + pub model: &'a str, + pub model_provider: &'a str, + pub sandbox_mode: SandboxMode, + /// Kept separate from sandbox mode because analytics reports network + /// capability as an independent resolved setting. + pub sandbox_network_access: bool, + /// Absent when the selected model/provider has no explicit effort setting. + pub reasoning_effort: Option, + /// None means no summary setting was resolved; Some(None) means summaries + /// were explicitly disabled. + pub reasoning_summary: Option, + pub service_tier: Option, + pub approval_policy: ApprovalPolicy, + pub approval_reviewer: ApprovalReviewer, + pub collaboration_mode: CollaborationMode, + /// Absent when no personality setting was resolved. + pub personality: Option, + pub is_first_turn: bool, +} + +/// Observation emitted when execution of a turn starts. +#[derive(Observation)] +#[observation(name = "turn.started", crate = "crate", uses = ["analytics"])] +pub struct TurnStarted<'a> { + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + pub config: TurnConfig<'a>, + + /// Unix timestamp in seconds when the turn started. + #[obs(level = "basic", class = "operational")] + pub started_at: i64, +} + +/// Token accounting reported for a completed turn. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +pub struct TurnTokenUsage { + pub input_tokens: i64, + pub cached_input_tokens: i64, + pub output_tokens: i64, + pub reasoning_output_tokens: i64, + pub total_tokens: i64, +} + +/// Observation emitted when a turn reaches a terminal state. +#[derive(Observation)] +#[observation(name = "turn.ended", crate = "crate", uses = ["analytics"])] +pub struct TurnEnded<'a> { + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + pub status: TurnStatus, + + /// Absent when a turn ends before provider usage is available. + #[obs(level = "basic", class = "operational")] + pub token_usage: Option, + + /// Unix timestamp in seconds when the turn ended. + #[obs(level = "basic", class = "operational")] + pub ended_at: i64, + + #[obs(level = "basic", class = "operational")] + pub duration_ms: i64, +} diff --git a/docs/observability-event-stream-design.md b/docs/observability-event-stream-design.md index 9a8e3969f3..2ca1bfd5b1 100644 --- a/docs/observability-event-stream-design.md +++ b/docs/observability-event-stream-design.md @@ -145,8 +145,8 @@ Example: use codex_observability::Observation; #[derive(Observation)] -#[observation(name = "turn.config_resolved", uses = ["analytics"])] -struct TurnConfigResolved<'a> { +#[observation(name = "turn.started", uses = ["analytics"])] +struct TurnStarted<'a> { #[obs(level = "basic", class = "identifier")] thread_id: &'a str, @@ -154,7 +154,7 @@ struct TurnConfigResolved<'a> { turn_id: &'a str, #[obs(level = "basic", class = "operational")] - model: &'a str, + started_at: i64, } ``` @@ -292,7 +292,7 @@ output schemas. | Observations | Analytics output | | --- | --- | | `thread.started` | `codex_thread_initialized` | -| `turn.started`, `turn.ended` with token usage, and future config observations | `codex_turn_event` | +| `turn.started` with resolved config, `turn.ended` with token usage | `codex_turn_event` | | `turn.steer_resolved` | `codex_turn_steer_event` | | compaction lifecycle observations | `codex_compaction_event` | | skill/app/plugin/guardian observations | existing feature events |