diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e0c399a3b2..ee66c82d3d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1382,6 +1382,7 @@ dependencies = [ "codex-app-server-protocol", "codex-git-utils", "codex-login", + "codex-observability", "codex-plugin", "codex-protocol", "codex-utils-absolute-path", @@ -2563,6 +2564,25 @@ dependencies = [ "url", ] +[[package]] +name = "codex-observability" +version = "0.0.0" +dependencies = [ + "codex-observability-derive", + "pretty_assertions", + "serde", + "serde_json", +] + +[[package]] +name = "codex-observability-derive" +version = "0.0.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "codex-ollama" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index ed6bdc0fd6..ca539d9150 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -48,6 +48,8 @@ members = [ "models-manager", "network-proxy", "ollama", + "observability", + "observability-derive", "process-hardening", "protocol", "realtime-webrtc", @@ -153,6 +155,8 @@ codex-model-provider-info = { path = "model-provider-info" } codex-models-manager = { path = "models-manager" } codex-network-proxy = { path = "network-proxy" } codex-ollama = { path = "ollama" } +codex-observability = { path = "observability" } +codex-observability-derive = { path = "observability-derive" } codex-otel = { path = "otel" } codex-plugin = { path = "plugin" } codex-model-provider = { path = "model-provider" } diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml index f706814d41..80df098b9a 100644 --- a/codex-rs/analytics/Cargo.toml +++ b/codex-rs/analytics/Cargo.toml @@ -16,6 +16,7 @@ workspace = true codex-app-server-protocol = { workspace = true } codex-git-utils = { workspace = true } codex-login = { workspace = true } +codex-observability = { workspace = true } codex-plugin = { workspace = true } codex-protocol = { workspace = true } os_info = { workspace = true } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 4b13b2819e..cce90cca8e 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -46,6 +46,7 @@ use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; use crate::facts::TurnSteerRequestError; use crate::facts::TurnTokenUsageFact; +use crate::observation_reducer::AnalyticsObservationReducer; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; @@ -1553,6 +1554,51 @@ async fn reducer_ingests_app_and_plugin_facts() { assert_eq!(payload[2]["event_type"], "codex_plugin_used"); } +#[tokio::test] +async fn app_used_observation_matches_legacy_analytics_fact() { + let mut legacy_reducer = AnalyticsReducer::default(); + let mut observation_reducer = AnalyticsObservationReducer::default(); + let mut legacy_events = Vec::new(); + let mut observation_events = Vec::new(); + + legacy_reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::AppUsed(AppUsedInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }, + app: AppInvocation { + connector_id: Some("drive".to_string()), + app_name: Some("Drive".to_string()), + invocation_type: Some(InvocationType::Implicit), + }, + })), + &mut legacy_events, + ) + .await; + + observation_reducer + .ingest_app_used( + codex_observability::events::AppUsed { + model_slug: "gpt-5", + thread_id: "thread-1", + turn_id: "turn-1", + connector_id: Some("drive"), + app_name: Some("Drive"), + invocation_type: Some(codex_observability::events::InvocationType::Implicit), + }, + &mut observation_events, + ) + .await; + + let legacy_payload = serde_json::to_value(&legacy_events).expect("serialize legacy events"); + let observation_payload = + serde_json::to_value(&observation_events).expect("serialize observation events"); + assert_eq!(observation_payload, legacy_payload); +} + #[tokio::test] async fn reducer_ingests_plugin_state_changed_fact() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 03f485a1c1..6b24006570 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -1,6 +1,7 @@ mod client; mod events; mod facts; +mod observation_reducer; mod reducer; use std::time::SystemTime; diff --git a/codex-rs/analytics/src/observation_reducer.rs b/codex-rs/analytics/src/observation_reducer.rs new file mode 100644 index 0000000000..cf660ec116 --- /dev/null +++ b/codex-rs/analytics/src/observation_reducer.rs @@ -0,0 +1,57 @@ +//! Projection from shared observations into analytics track events. +//! +//! This module deliberately reuses the existing analytics reducer while the +//! shared observation stream is being introduced. That gives conformance tests +//! a small, stable bridge: one side feeds legacy analytics facts and the other +//! feeds typed observations, then both paths must produce identical track +//! requests. + +use crate::events::TrackEventRequest; +use crate::facts; +use crate::facts::AnalyticsFact; +use crate::facts::AppInvocation; +use crate::facts::AppUsedInput; +use crate::facts::CustomAnalyticsFact; +use crate::facts::TrackEventsContext; +use crate::reducer::AnalyticsReducer; +use codex_observability::events; + +/// Analytics reducer entrypoint for typed observations. +#[derive(Default)] +pub(crate) struct AnalyticsObservationReducer { + legacy: AnalyticsReducer, +} + +impl AnalyticsObservationReducer { + /// Ingests an app.used observation and emits the current analytics event. + pub(crate) async fn ingest_app_used( + &mut self, + observation: events::AppUsed<'_>, + out: &mut Vec, + ) { + let tracking = TrackEventsContext { + model_slug: observation.model_slug.to_string(), + thread_id: observation.thread_id.to_string(), + turn_id: observation.turn_id.to_string(), + }; + let app = AppInvocation { + connector_id: observation.connector_id.map(str::to_string), + app_name: observation.app_name.map(str::to_string), + invocation_type: observation.invocation_type.map(map_invocation_type), + }; + + self.legacy + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::AppUsed(AppUsedInput { tracking, app })), + out, + ) + .await; + } +} + +fn map_invocation_type(invocation_type: events::InvocationType) -> facts::InvocationType { + match invocation_type { + events::InvocationType::Explicit => facts::InvocationType::Explicit, + events::InvocationType::Implicit => facts::InvocationType::Implicit, + } +} diff --git a/codex-rs/observability-derive/BUILD.bazel b/codex-rs/observability-derive/BUILD.bazel new file mode 100644 index 0000000000..ee0df71583 --- /dev/null +++ b/codex-rs/observability-derive/BUILD.bazel @@ -0,0 +1,7 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "observability-derive", + crate_name = "codex_observability_derive", + proc_macro = True, +) diff --git a/codex-rs/observability-derive/Cargo.toml b/codex-rs/observability-derive/Cargo.toml new file mode 100644 index 0000000000..1fba8b8485 --- /dev/null +++ b/codex-rs/observability-derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-observability-derive" +version.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full", "extra-traits"] } + +[lints] +workspace = true diff --git a/codex-rs/observability-derive/src/lib.rs b/codex-rs/observability-derive/src/lib.rs new file mode 100644 index 0000000000..0f8412e02d --- /dev/null +++ b/codex-rs/observability-derive/src/lib.rs @@ -0,0 +1,208 @@ +//! Derive macro implementation for `codex-observability`. +//! +//! The macro is intentionally small: it turns annotated struct fields into +//! `ObservationFieldVisitor` calls and refuses to compile if any exported field +//! is missing policy metadata. Filtering, serialization, redaction, and +//! destination-specific mapping stay in sinks/reducers. + +use proc_macro::TokenStream; +use quote::format_ident; +use quote::quote; +use syn::Data; +use syn::DeriveInput; +use syn::Expr; +use syn::Fields; +use syn::LitStr; +use syn::Path; +use syn::parse_macro_input; + +/// Derives `codex_observability::Observation` for a named struct. +/// +/// Required attributes: +/// +/// - `#[observation(name = "domain.event")]` on the struct. +/// - `#[obs(level = "basic|detailed|trace", class = "...")]` on every field. +/// +/// Event definitions inside `codex-observability` itself may use +/// `#[observation(crate = "crate")]` so generated code refers to local types +/// instead of the externally visible crate path. +#[proc_macro_derive(Observation, attributes(observation, obs))] +pub fn derive_observation(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + expand_observation(input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} + +fn expand_observation(input: DeriveInput) -> syn::Result { + let name = input.ident; + let observation_attr = observation_attr(&input.attrs)?; + let event_name = observation_attr.name; + let crate_path = observation_attr.crate_path; + let Data::Struct(data) = input.data else { + return Err(syn::Error::new_spanned( + name, + "Observation can only be derived for structs", + )); + }; + let Fields::Named(fields) = data.fields else { + return Err(syn::Error::new_spanned( + name, + "Observation requires a struct with named fields", + )); + }; + + let mut visits = Vec::new(); + for field in fields.named { + let Some(field_name) = field.ident else { + continue; + }; + let meta = obs_meta(&field.attrs, &field_name, &crate_path)?; + let field_name_lit = LitStr::new(&field_name.to_string(), field_name.span()); + visits.push(quote! { + visitor.field(#field_name_lit, #meta, &self.#field_name); + }); + } + + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + Ok(quote! { + impl #impl_generics #crate_path::Observation for #name #ty_generics #where_clause { + const NAME: &'static str = #event_name; + + fn visit_fields(&self, visitor: &mut V) { + #(#visits)* + } + } + }) +} + +struct ObservationAttr { + name: LitStr, + crate_path: Path, +} + +fn observation_attr(attrs: &[syn::Attribute]) -> syn::Result { + for attr in attrs { + if !attr.path().is_ident("observation") { + continue; + } + let mut name = None; + let mut crate_path = None; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("name") { + name = Some(meta.value()?.parse::()?); + Ok(()) + } else if meta.path.is_ident("crate") { + let value = meta.value()?.parse::()?; + // The override is needed only when deriving observations from + // inside `codex-observability`, where the external crate name is + // not available. Ordinary users get the stable public path. + crate_path = Some(value.parse::()?); + Ok(()) + } else { + Err(meta.error("unsupported observation attribute")) + } + })?; + if let Some(name) = name { + return Ok(ObservationAttr { + name, + crate_path: crate_path.unwrap_or_else(|| syn::parse_quote!(::codex_observability)), + }); + } + } + Err(syn::Error::new( + proc_macro2::Span::call_site(), + "missing #[observation(name = \"...\")]", + )) +} + +fn obs_meta( + attrs: &[syn::Attribute], + field_name: &syn::Ident, + crate_path: &Path, +) -> syn::Result { + for attr in attrs { + if !attr.path().is_ident("obs") { + continue; + } + let mut level = None; + let mut class = None; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("level") { + level = Some(meta.value()?.parse::()?); + Ok(()) + } else if meta.path.is_ident("class") { + class = Some(meta.value()?.parse::()?); + Ok(()) + } else { + Err(meta.error("unsupported obs attribute")) + } + })?; + let level = level.ok_or_else(|| { + syn::Error::new_spanned(attr, "missing obs level, expected level = \"...\"") + })?; + let class = class.ok_or_else(|| { + syn::Error::new_spanned(attr, "missing obs class, expected class = \"...\"") + })?; + let detail = detail_expr(&level, crate_path)?; + let data_class = data_class_expr(&class, crate_path)?; + return Ok(quote! { + #crate_path::FieldMeta::new(#detail, #data_class) + }); + } + Err(syn::Error::new_spanned( + field_name, + "missing #[obs(level = \"...\", class = \"...\")]", + )) +} + +fn detail_expr(value: &LitStr, crate_path: &Path) -> syn::Result { + enum_expr( + value, + "detail level", + &[ + ("basic", "Basic"), + ("detailed", "Detailed"), + ("trace", "Trace"), + ], + quote!(#crate_path::DetailLevel), + ) +} + +fn data_class_expr(value: &LitStr, crate_path: &Path) -> syn::Result { + enum_expr( + value, + "data class", + &[ + ("identifier", "Identifier"), + ("operational", "Operational"), + ("environment", "Environment"), + ("content", "Content"), + ("secret_risk", "SecretRisk"), + ], + quote!(#crate_path::DataClass), + ) +} + +fn enum_expr( + value: &LitStr, + label: &str, + variants: &[(&str, &str)], + path: proc_macro2::TokenStream, +) -> syn::Result { + let raw = value.value(); + let Some((_, variant)) = variants.iter().find(|(name, _)| *name == raw) else { + let expected = variants + .iter() + .map(|(name, _)| format!("\"{name}\"")) + .collect::>() + .join(", "); + return Err(syn::Error::new_spanned( + value, + format!("invalid {label} {raw:?}, expected one of {expected}"), + )); + }; + let variant = format_ident!("{variant}"); + let expr: Expr = syn::parse_quote!(#path::#variant); + Ok(quote!(#expr)) +} diff --git a/codex-rs/observability/BUILD.bazel b/codex-rs/observability/BUILD.bazel new file mode 100644 index 0000000000..17ee1c0e3c --- /dev/null +++ b/codex-rs/observability/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "observability", + crate_name = "codex_observability", +) diff --git a/codex-rs/observability/Cargo.toml b/codex-rs/observability/Cargo.toml new file mode 100644 index 0000000000..434588716f --- /dev/null +++ b/codex-rs/observability/Cargo.toml @@ -0,0 +1,21 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-observability" +version.workspace = true + +[lib] +doctest = true +name = "codex_observability" +path = "src/lib.rs" + +[dependencies] +codex-observability-derive = { workspace = true } +serde = { workspace = true, features = ["derive"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/observability/src/events.rs b/codex-rs/observability/src/events.rs new file mode 100644 index 0000000000..cdcaba2701 --- /dev/null +++ b/codex-rs/observability/src/events.rs @@ -0,0 +1,43 @@ +//! Canonical Codex observation event definitions. + +use crate::Observation; +use serde::Serialize; + +/// How an app/tool/plugin capability was selected by the user or system. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum InvocationType { + /// The user explicitly mentioned or selected the capability. + Explicit, + /// Codex inferred that the capability should be used. + Implicit, +} + +/// Observation emitted when Codex uses an app connector during a turn. +#[derive(Observation)] +#[observation(name = "app.used", crate = "crate")] +pub struct AppUsed<'a> { + /// Model slug active for the turn where the app was used. + #[obs(level = "basic", class = "operational")] + pub model_slug: &'a str, + + /// Thread that owns the turn. + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + /// Turn where the app was used. + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + /// Stable connector identifier when the app is backed by a connector. + #[obs(level = "basic", class = "identifier")] + pub connector_id: Option<&'a str>, + + /// User-facing app name. + #[obs(level = "basic", class = "operational")] + pub app_name: Option<&'a str>, + + /// Whether usage was explicit or implicit. + #[obs(level = "basic", class = "operational")] + pub invocation_type: Option, +} diff --git a/codex-rs/observability/src/lib.rs b/codex-rs/observability/src/lib.rs new file mode 100644 index 0000000000..bef7314bed --- /dev/null +++ b/codex-rs/observability/src/lib.rs @@ -0,0 +1,127 @@ +//! Shared semantic observation API for Codex runtime facts. +//! +//! Observations describe facts that occurred in Codex. Destination-specific +//! systems such as analytics, rollout trace, OTEL events, and OTEL metrics +//! consume observations through sinks and reducers. +//! +//! Field metadata is required because sinks must decide whether they may read a +//! field before serializing or exporting it. Missing field annotations are a +//! compile-time error: +//! +//! ```compile_fail +//! use codex_observability::Observation; +//! +//! #[derive(Observation)] +//! #[observation(name = "example.missing_field_meta")] +//! struct MissingFieldMeta { +//! field: &'static str, +//! } +//! ``` +//! +//! Observation names are also required: +//! +//! ```compile_fail +//! use codex_observability::Observation; +//! +//! #[derive(Observation)] +//! struct MissingObservationName { +//! #[obs(level = "basic", class = "operational")] +//! field: &'static str, +//! } +//! ``` + +pub mod events; + +pub use codex_observability_derive::Observation; +use serde::Serialize; + +/// A runtime fact emitted by Codex. +/// +/// Implementations visit every exported field together with its field metadata. +/// Sinks use that metadata to apply destination-specific policy before +/// serialization, storage, or export. +pub trait Observation { + /// Stable semantic event name, for example `turn.started`. + const NAME: &'static str; + + /// Visits the fields that make up this observation. + fn visit_fields(&self, visitor: &mut V); +} + +/// Receives observation fields after policy metadata has been attached. +/// +/// Implementations should inspect `meta` before serializing `value`. This keeps +/// remote sinks from accidentally materializing local-only content fields. +pub trait ObservationFieldVisitor { + /// Visits one field from an observation. + fn field(&mut self, name: &'static str, meta: FieldMeta, value: &T); +} + +/// Consumes observations. +/// +/// A sink may serialize immediately, reduce into another event shape, or fan +/// out to additional sinks. The trait is generic so callers can pass borrowed +/// typed observations without allocating an intermediate event object. +pub trait ObservationSink { + /// Observes a single typed event. + fn observe(&self, event: &E); +} + +/// Policy metadata attached to a single observation field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FieldMeta { + /// How much detail a sink must be allowed to read before consuming the field. + pub detail: DetailLevel, + /// Semantic/privacy class for the field. + pub class: DataClass, +} + +impl FieldMeta { + /// Creates metadata for a field. + pub const fn new(detail: DetailLevel, class: DataClass) -> Self { + Self { detail, class } + } +} + +/// Coarse detail level for a field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DetailLevel { + /// Lifecycle, identifiers, status, counts, model/config, and timing. + Basic, + /// Bounded previews and richer runtime summaries. + Detailed, + /// Raw or near-raw diagnostic evidence intended for local traces. + Trace, +} + +/// Semantic/privacy class for a field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DataClass { + /// Thread IDs, turn IDs, call IDs, and similar correlation identifiers. + Identifier, + /// Status, duration, model, provider, token counts, and tool kind. + Operational, + /// Working directory, repository, OS/runtime, or client metadata. + Environment, + /// User text, assistant text, command text, tool output, or model payloads. + Content, + /// Headers, environment values, auth-like payloads, or raw request blobs. + SecretRisk, +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn field_meta_preserves_detail_and_class() { + assert_eq!( + FieldMeta::new(DetailLevel::Trace, DataClass::Content), + FieldMeta { + detail: DetailLevel::Trace, + class: DataClass::Content, + } + ); + } +} diff --git a/codex-rs/observability/tests/derive.rs b/codex-rs/observability/tests/derive.rs new file mode 100644 index 0000000000..201b15433f --- /dev/null +++ b/codex-rs/observability/tests/derive.rs @@ -0,0 +1,81 @@ +use codex_observability::DataClass; +use codex_observability::DetailLevel; +use codex_observability::FieldMeta; +use codex_observability::Observation; +use codex_observability::ObservationFieldVisitor; +use pretty_assertions::assert_eq; +use serde_json::Value; + +#[derive(Observation)] +#[observation(name = "turn.config_resolved")] +struct TurnConfigResolved<'a> { + #[obs(level = "basic", class = "identifier")] + thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + model: &'a str, +} + +#[derive(Debug, PartialEq)] +struct CapturedField { + name: &'static str, + meta: FieldMeta, + value: Value, +} + +#[derive(Default)] +struct CapturingVisitor { + fields: Vec, +} + +impl ObservationFieldVisitor for CapturingVisitor { + fn field( + &mut self, + name: &'static str, + meta: FieldMeta, + value: &T, + ) { + let value = match serde_json::to_value(value) { + Ok(value) => value, + Err(err) => panic!("field should serialize: {err}"), + }; + self.fields.push(CapturedField { name, meta, value }); + } +} + +#[test] +fn derive_visits_annotated_fields_with_metadata() { + let event = TurnConfigResolved { + thread_id: "thread-1", + turn_id: "turn-1", + model: "gpt-5.4", + }; + + let mut visitor = CapturingVisitor::default(); + event.visit_fields(&mut visitor); + + assert_eq!(TurnConfigResolved::NAME, "turn.config_resolved"); + assert_eq!( + visitor.fields, + vec![ + CapturedField { + name: "thread_id", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Identifier), + value: Value::String("thread-1".to_string()), + }, + CapturedField { + name: "turn_id", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Identifier), + value: Value::String("turn-1".to_string()), + }, + CapturedField { + name: "model", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Operational), + value: Value::String("gpt-5.4".to_string()), + }, + ] + ); +} diff --git a/docs/observability-event-stream-design.md b/docs/observability-event-stream-design.md new file mode 100644 index 0000000000..ab84cb883a --- /dev/null +++ b/docs/observability-event-stream-design.md @@ -0,0 +1,417 @@ +# Observability Event Stream + +Codex currently has three overlapping observability systems: + +- `codex-rs/analytics` reduces runtime facts into remote product analytics. +- `codex-rs/rollout-trace` records rich local diagnostic traces and reduces + them into a replay/debug graph. +- `codex-rs/otel` emits logs, trace-safe events, and metrics, while native + `tracing` spans carry execution scope and W3C trace context. + +The overlap is real: thread lifecycle, turns, model calls, tools, compaction, +subagents, transport, and feature usage are described more than once. Directly +merging the crates would couple incompatible outputs. The shared layer should +instead define the facts Codex emits; each destination should decide how those +facts are filtered, reduced, stored, or exported. + +## Decision + +Introduce a shared observation stream for semantic facts. + +```text +Observations own semantic facts and metrics derived from facts. +tracing owns spans and trace context propagation. +OTEL is a consumer of observations, not a second event taxonomy. +``` + +This gives one rule for callers: + +- Use observations when a thing happened. +- Use sinks/reducers for analytics, rollout trace, OTEL events, OTEL metrics, + local buffers, and feedback upload. +- Use `tracing::span!` when code needs active execution scope, parent/child + causality, span mutation, or W3C context propagation. + +The target state should not be a long-lived mix where some facts use +observations and other facts call analytics or OTEL directly. Direct OTEL code +should remain for exporter setup, native spans, context propagation, and +low-level tracing plumbing. + +## Goals + +- Define canonical observation events that describe what Codex did, not where + the data is going. +- Keep analytics, rollout trace, OTEL logs, and OTEL metrics as projections. +- Apply filtering at field level so a single observation can contain both safe + aggregate fields and richer local-only evidence. +- Preserve existing analytics and OTEL output behavior while moving callsites + toward the shared stream. +- Support local rich retention for feedback/debug upload with explicit policy. +- Prove analytics equivalence with side-by-side conformance tests. + +## Non-Goals + +- Do not make analytics facts the source of truth for rollout trace. +- Do not make rollout trace raw events the source of truth for analytics. +- Do not encode downstream event names such as `codex_turn_event` or + `codex.tool_result` into the canonical taxonomy. +- Do not replace native spans with `SpanStarted` / `SpanEnded` observations. +- Do not require full rollout trace graph equivalence before the first version; + rollout trace is experimental and can use lighter validation initially. + +## Shape + +```text +codex-core / app-server + emit observations + +codex-observability + observation traits, field metadata, sink policy, event definitions + +codex-observability-derive + derive macro for field metadata and visitors + +sinks/reducers + analytics projection -> TrackEventRequest + rollout trace projection -> local trace bundle + reduced graph + OTEL event projection -> current log and trace-safe event names + OTEL metric projection -> current counters and histograms + ringbuffer -> recent rich local observations + +native tracing spans + remain direct tracing instrumentation +``` + +The derive crate exists because Rust does not expose struct field attributes at +runtime. The public API can still re-export the derive from +`codex-observability` so most callsites only import one crate. + +## Why Not Just OTEL? + +An alternative is to emit semantic facts as ordinary `tracing` events and let a +local subscriber, OTEL exporter, analytics reducer, or trace reducer all consume +the same event stream. That is attractive because Codex already uses +`tracing`, and local JSON event capture is easy to wire up. + +We should still avoid making OTEL/tracing events the canonical schema: + +- `tracing` fields are stringly typed at the reducer boundary. Renames or type + changes become runtime reducer failures instead of Rust compile errors. +- OTEL trace-safe export has a different privacy posture than local diagnostic + traces. Rich local fields such as tool output, prompts, model payloads, and + terminal output should not share a target with remotely exported span events. +- Field-level policy needs metadata that `tracing` does not preserve in a + structured way. We need to know both detail level and data class before a + sink serializes a field. +- Analytics conformance is easier from typed observations than from flattened + JSON logs, because reducers can match on Rust event types and fields. +- Spans and semantic facts have different lifecycles. Spans model active scope + and context propagation; observations model facts that happened. + +OTEL should therefore consume observations for semantic events and derived +metrics, while native `tracing` remains the right tool for spans, context +propagation, and exporter plumbing. A local trace sink can still write JSONL +bundles; it should write observation envelopes rather than raw +`tracing_subscriber` event JSON. + +## Observation API + +The shared schema API can be small: + +```rust +pub trait Observation { + const NAME: &'static str; + + fn visit_fields(&self, visitor: &mut V); +} + +pub trait ObservationFieldVisitor { + fn field( + &mut self, + name: &'static str, + meta: FieldMeta, + value: &T, + ); +} + +pub trait ObservationSink { + fn observe(&self, event: &E); +} +``` + +Example: + +```rust +use codex_observability::Observation; + +#[derive(Observation)] +#[observation(name = "turn.config_resolved")] +struct TurnConfigResolved<'a> { + #[obs(level = "basic", class = "identifier")] + thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + model: &'a str, +} +``` + +The derive only exposes metadata. It should not implement analytics mapping, +trace persistence, redaction, schema generation, or export behavior. + +## Callsite API + +Product/runtime code should usually call semantic helper methods rather than +inline-constructing large observation structs. Direct struct construction is +fine in tests, very small leaf events, or helper internals, but it should not be +the normal instrumentation style. + +The helper layer should do the tedious, failure-prone work near the callsite: +extract common IDs, join session/thread/turn context, compute durations, +normalize status and error fields, avoid trace-only allocations when no sink +can read them, and then emit the typed observation. + +Example shape: + +```rust +observability.turn().config_resolved(&session, &turn_context, &input).await; +observability.tool().ended(&invocation, status, &response); +observability.transport().api_request_completed(&request, &outcome); +``` + +This mirrors the useful part of the existing systems: + +- rollout trace has `RolloutTraceRecorder::record_*` helpers +- analytics has `AnalyticsEventsClient::track_*` helpers +- OTEL has `SessionTelemetry::record_*` helpers + +The helper layer must not become a second taxonomy. It is ergonomic glue over +canonical observation structs. Prefer small domain-specific helper groups such +as turn, tool, inference, transport, compaction, and feature usage over one +large recorder with every method in the system. + +## Rich Payloads + +Start simple. Ordinary observation fields should be eager values or borrowed +references. Sinks should reject fields by metadata before serializing them, so +analytics and OTEL metrics do not inspect trace/content fields. + +Some trace fields point at large objects that usually already exist at the +callsite: model requests, model responses, tool invocations, tool results, or +terminal output. Prefer passing borrowed serializable views of those objects to +the observation helper rather than building `serde_json::Value` eagerly. + +Do not introduce a generalized lazy-field system in the first implementation. +Closures such as `|| build_trace_payload(...)` are useful when constructing the +trace view itself is expensive, but they add API complexity. Treat them as a +later optimization for measured hot spots, not as the default observation +style. + +## Field Policy + +Filtering must be field-level. Event-level labels such as basic/detailed/trace +are too coarse because one event often contains both remote-safe counters and +local-only content. + +Each field should carry at least: + +- **Detail level**: `basic`, `detailed`, or `trace`. +- **Data class**: `identifier`, `operational`, `environment`, `content`, or + `secret_risk`. + +Detail level is not privacy by itself. A tiny field can still be unsafe for +remote export, and a trace-level field can be trace-level because it is large +rather than sensitive. Sinks must filter by both axes. + +Expected sink policies: + +- Analytics allows basic identifiers/operational fields plus selected + environment fields; it denies content and secret-risk fields. +- Rollout trace allows rich local fields, with explicit redaction rules for + secret-risk material. +- OTEL logs preserve today's log export behavior, including account/email only + where today's policy allows them. +- OTEL trace-safe events prefer lengths, counts, status, timing, and coarse + categories over content. +- OTEL metrics read only the dimensions needed for existing metric names/tags. +- Feedback upload applies an explicit user-approved policy over the ringbuffer. + +## Event Taxonomy + +The taxonomy should be designed from Codex workflows, not from existing +destination schemas. Names should describe facts that occurred in the system: +`turn.ended`, `tool_call.ended`, `transport.api_request_completed`. + +Rules: + +- Prefer stable domain facts over analytics, OTEL, or trace storage names. +- Model lifecycle facts explicitly when downstream reducers need ordering or + duration. +- Keep event count small until a consumer needs a distinct fact. +- Put richness in fields, not in parallel event variants. +- Add transport/runtime facts when they are first-class telemetry today, even + if analytics ignores them. +- Keep an escape hatch only for local experimental tracing, not remote export. + +Initial workflow coverage should be chosen by conformance need: + +| Workflow | Canonical examples | Primary consumers | +| --- | --- | --- | +| Session/thread | `session.config_resolved`, `thread.started`, `thread.ended` | analytics, OTEL, rollout | +| Turn lifecycle | `turn.requested`, `turn.started`, `turn.ended`, `turn.token_usage_observed` | analytics, OTEL, rollout | +| Turn timing | `turn.first_token_observed`, `turn.first_message_observed` | OTEL metrics, rollout | +| Model I/O | `inference.started`, `inference.sse_event_observed`, `inference.completed`, `inference.failed` | OTEL, rollout | +| Tools | `tool_call.started`, `tool_call.approval_resolved`, `tool_call.ended` | OTEL, rollout | +| Compaction | `compaction.started`, `compaction.installed`, `compaction.ended` | analytics, rollout | +| Agents | `agent.task_sent`, `agent.message_sent`, `agent.result_delivered`, `agent.closed` | rollout, analytics subset | +| Product features | `skill.invoked`, `app.used`, `plugin.state_changed`, `guardian.review_completed` | analytics | +| Transport/auth | `transport.api_request_completed`, `transport.websocket_request_completed`, `auth.recovery_step_completed` | OTEL | + +This table is not a complete schema. Each event still needs a typed Rust +definition with field annotations before implementation. + +## Projections + +### Analytics + +Analytics becomes a reducer over observations. Existing product events remain +output schemas. + +| Observations | Analytics output | +| --- | --- | +| `thread.started` | `codex_thread_initialized` | +| turn config/lifecycle/token observations | `codex_turn_event` | +| `turn.steer_resolved` | `codex_turn_steer_event` | +| compaction lifecycle observations | `codex_compaction_event` | +| skill/app/plugin/guardian observations | existing feature events | + +The analytics reducer continues to own batching, deduplication, connection +metadata joins, and product event naming. + +### Rollout Trace + +Rollout trace consumes observations and decides which fields become trace +payloads. Trace storage details such as `RawPayloadRef` should remain a trace +implementation detail, not part of the shared event definitions. + +Example: `inference.completed` can include response material as a trace-level +field. The trace sink may write it to `payloads/*.json`; analytics and OTEL +metrics never read it. + +### OTEL + +OTEL consumes observations for every semantic fact and derived metric. The +projection preserves existing exported names while removing separate callsite +event definitions. + +| Observations | OTEL output | +| --- | --- | +| session/thread observations | `codex.conversation_starts`, `codex.thread.started` | +| `turn.input_received` | `codex.user_prompt` | +| `tool_call.approval_resolved` | `codex.tool_decision` | +| `tool_call.ended` | `codex.tool_result`, tool count/duration metrics | +| transport observations | API/websocket events and count/duration metrics | +| `auth.recovery_step_completed` | `codex.auth_recovery` | +| inference/SSE observations | `codex.sse_event`, SSE duration, response timing metrics | +| turn timing/token observations | TTFT, TTFM, E2E duration, token usage metrics | + +Log-only and trace-safe OTEL variants can be emitted from the same observation +by applying different field policies. + +## Conformance Testing + +The highest-value equivalence target is analytics because it is existing +product behavior. + +```text +same E2E Codex run + legacy analytics facts -> current AnalyticsReducer -> TrackEventRequest JSON + observations -> analytics observation reducer -> TrackEventRequest JSON + +assert exact equality after stable JSON normalization +``` + +At least one conformance path should apply the analytics field policy so bad +or missing annotations fail tests. Recommended first scenarios: + +- thread start +- normal turn +- failed or interrupted turn +- accepted and rejected turn steering +- compaction +- skill, app, plugin, and guardian events +- subagent thread start + +Rollout trace can start with observation-to-trace unit tests plus one or two +end-to-end smoke tests that write and reduce a bundle. Full graph equivalence +is not necessary for the initial experimental rollout feature. + +OTEL should get projection tests that compare exported event names, fields, +metric names, and tags for representative observations. Native span tests +should remain in the tracing layer. + +## Implementation Stages + +Build the proof of life on a fresh `main` branch, but keep commits separable so +the working branch can later be split into reviewable PRs. +Each stage should leave a small, documented API surface and a passing local +verification gate. Non-essential behavior should stay as a TODO until a later +stage needs it. + +1. **Observation foundation** + - Add `codex-observability` and `codex-observability-derive`. + - Verify with crate-local tests that derived observations expose field + metadata and reject missing annotations at compile time. + - Run: `cargo test -p codex-observability`. + +2. **Analytics conformance slice** + - Add the first typed observations and an analytics projection for one + existing analytics flow. + - Verify legacy analytics facts and observation-derived facts produce + identical `TrackEventRequest` JSON after stable normalization. + - Run: targeted `cargo test -p codex-analytics`. + +3. **E2E analytics shadowing** + - Capture legacy facts and observations from the same core/app-server test + run for selected scenarios. + - Verify exact analytics output equality for thread start, normal turn, + failed/interrupted turn, steering, compaction, and one feature event. + - Run the targeted core/app-server integration tests added for each + scenario. + +4. **Rollout trace projection** + - Add a rollout trace sink that consumes the same observations for + thread/turn, inference, and tool lifecycle. + - Verify with observation-to-trace unit tests and one smoke test that writes + and reduces a local bundle. + +5. **OTEL projection proof** + - Add a small OTEL projection for a high-value event such as tool result. + - Verify exported event names, field policy, metric names, and tags match + the current behavior. + +## Migration + +Start with a shadow observation stream and conformance tests. Once analytics +equivalence is reliable, move callsites from direct analytics facts to +observations. + +Rollout trace can adopt observations directly because it is experimental. + +OTEL should be migrated by category rather than left permanently mixed: +tool/user/turn events, transport events, auth events, and derived metrics can +move behind the OTEL projection. Native spans and trace context stay where they +are. + +## Open Questions + +- What is the smallest useful field-class set? +- Should timestamps and sequence numbers live in the observation envelope or be + supplied by sinks? +- Where should fanout live so callsites do not grow `codex-core` further? +- How should borrowed trace-level payloads avoid hot-path allocations while + still allowing sinks to serialize when policy permits? +- Which direct OTEL counters are first-class observations, and which are + metrics derived from broader observations?