From 50e23d92c6bf9ef5018e1f86eed01c332d75c6e5 Mon Sep 17 00:00:00 2001 From: Albin Cassirer Date: Sun, 19 Apr 2026 14:53:03 -0700 Subject: [PATCH] [observability] Add shared observation foundation Introduce a small codex-observability crate with typed observations, field-level policy metadata, and a derive macro that requires each exported field to declare detail level and data class. The derive crate stays narrow: it only generates visitor calls and leaves filtering, serialization, and destination-specific mapping to sinks and reducers. Add the first canonical app.used observation and a private analytics observation reducer that projects it through the existing AnalyticsReducer. This gives us a side-by-side conformance test where the legacy analytics fact path and the observation path must emit identical TrackEventRequest JSON. Document the target architecture and staged verification plan so later work can expand the taxonomy, field policy, E2E shadowing, rollout trace, and OTEL projection without widening this first checkpoint. --- codex-rs/Cargo.lock | 20 + codex-rs/Cargo.toml | 4 + codex-rs/analytics/Cargo.toml | 1 + .../analytics/src/analytics_client_tests.rs | 46 ++ codex-rs/analytics/src/lib.rs | 1 + codex-rs/analytics/src/observation_reducer.rs | 57 +++ codex-rs/observability-derive/BUILD.bazel | 7 + codex-rs/observability-derive/Cargo.toml | 16 + codex-rs/observability-derive/src/lib.rs | 208 +++++++++ codex-rs/observability/BUILD.bazel | 6 + codex-rs/observability/Cargo.toml | 21 + codex-rs/observability/src/events.rs | 43 ++ codex-rs/observability/src/lib.rs | 127 ++++++ codex-rs/observability/tests/derive.rs | 81 ++++ docs/observability-event-stream-design.md | 417 ++++++++++++++++++ 15 files changed, 1055 insertions(+) create mode 100644 codex-rs/analytics/src/observation_reducer.rs create mode 100644 codex-rs/observability-derive/BUILD.bazel create mode 100644 codex-rs/observability-derive/Cargo.toml create mode 100644 codex-rs/observability-derive/src/lib.rs create mode 100644 codex-rs/observability/BUILD.bazel create mode 100644 codex-rs/observability/Cargo.toml create mode 100644 codex-rs/observability/src/events.rs create mode 100644 codex-rs/observability/src/lib.rs create mode 100644 codex-rs/observability/tests/derive.rs create mode 100644 docs/observability-event-stream-design.md diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index e0c399a3b2..ee66c82d3d 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -1382,6 +1382,7 @@ dependencies = [ "codex-app-server-protocol", "codex-git-utils", "codex-login", + "codex-observability", "codex-plugin", "codex-protocol", "codex-utils-absolute-path", @@ -2563,6 +2564,25 @@ dependencies = [ "url", ] +[[package]] +name = "codex-observability" +version = "0.0.0" +dependencies = [ + "codex-observability-derive", + "pretty_assertions", + "serde", + "serde_json", +] + +[[package]] +name = "codex-observability-derive" +version = "0.0.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "codex-ollama" version = "0.0.0" diff --git a/codex-rs/Cargo.toml b/codex-rs/Cargo.toml index ed6bdc0fd6..ca539d9150 100644 --- a/codex-rs/Cargo.toml +++ b/codex-rs/Cargo.toml @@ -48,6 +48,8 @@ members = [ "models-manager", "network-proxy", "ollama", + "observability", + "observability-derive", "process-hardening", "protocol", "realtime-webrtc", @@ -153,6 +155,8 @@ codex-model-provider-info = { path = "model-provider-info" } codex-models-manager = { path = "models-manager" } codex-network-proxy = { path = "network-proxy" } codex-ollama = { path = "ollama" } +codex-observability = { path = "observability" } +codex-observability-derive = { path = "observability-derive" } codex-otel = { path = "otel" } codex-plugin = { path = "plugin" } codex-model-provider = { path = "model-provider" } diff --git a/codex-rs/analytics/Cargo.toml b/codex-rs/analytics/Cargo.toml index f706814d41..80df098b9a 100644 --- a/codex-rs/analytics/Cargo.toml +++ b/codex-rs/analytics/Cargo.toml @@ -16,6 +16,7 @@ workspace = true codex-app-server-protocol = { workspace = true } codex-git-utils = { workspace = true } codex-login = { workspace = true } +codex-observability = { workspace = true } codex-plugin = { workspace = true } codex-protocol = { workspace = true } os_info = { workspace = true } diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index 4b13b2819e..cce90cca8e 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -46,6 +46,7 @@ use crate::facts::TurnResolvedConfigFact; use crate::facts::TurnStatus; use crate::facts::TurnSteerRequestError; use crate::facts::TurnTokenUsageFact; +use crate::observation_reducer::AnalyticsObservationReducer; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; @@ -1553,6 +1554,51 @@ async fn reducer_ingests_app_and_plugin_facts() { assert_eq!(payload[2]["event_type"], "codex_plugin_used"); } +#[tokio::test] +async fn app_used_observation_matches_legacy_analytics_fact() { + let mut legacy_reducer = AnalyticsReducer::default(); + let mut observation_reducer = AnalyticsObservationReducer::default(); + let mut legacy_events = Vec::new(); + let mut observation_events = Vec::new(); + + legacy_reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::AppUsed(AppUsedInput { + tracking: TrackEventsContext { + model_slug: "gpt-5".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + }, + app: AppInvocation { + connector_id: Some("drive".to_string()), + app_name: Some("Drive".to_string()), + invocation_type: Some(InvocationType::Implicit), + }, + })), + &mut legacy_events, + ) + .await; + + observation_reducer + .ingest_app_used( + codex_observability::events::AppUsed { + model_slug: "gpt-5", + thread_id: "thread-1", + turn_id: "turn-1", + connector_id: Some("drive"), + app_name: Some("Drive"), + invocation_type: Some(codex_observability::events::InvocationType::Implicit), + }, + &mut observation_events, + ) + .await; + + let legacy_payload = serde_json::to_value(&legacy_events).expect("serialize legacy events"); + let observation_payload = + serde_json::to_value(&observation_events).expect("serialize observation events"); + assert_eq!(observation_payload, legacy_payload); +} + #[tokio::test] async fn reducer_ingests_plugin_state_changed_fact() { let mut reducer = AnalyticsReducer::default(); diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 03f485a1c1..6b24006570 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -1,6 +1,7 @@ mod client; mod events; mod facts; +mod observation_reducer; mod reducer; use std::time::SystemTime; diff --git a/codex-rs/analytics/src/observation_reducer.rs b/codex-rs/analytics/src/observation_reducer.rs new file mode 100644 index 0000000000..cf660ec116 --- /dev/null +++ b/codex-rs/analytics/src/observation_reducer.rs @@ -0,0 +1,57 @@ +//! Projection from shared observations into analytics track events. +//! +//! This module deliberately reuses the existing analytics reducer while the +//! shared observation stream is being introduced. That gives conformance tests +//! a small, stable bridge: one side feeds legacy analytics facts and the other +//! feeds typed observations, then both paths must produce identical track +//! requests. + +use crate::events::TrackEventRequest; +use crate::facts; +use crate::facts::AnalyticsFact; +use crate::facts::AppInvocation; +use crate::facts::AppUsedInput; +use crate::facts::CustomAnalyticsFact; +use crate::facts::TrackEventsContext; +use crate::reducer::AnalyticsReducer; +use codex_observability::events; + +/// Analytics reducer entrypoint for typed observations. +#[derive(Default)] +pub(crate) struct AnalyticsObservationReducer { + legacy: AnalyticsReducer, +} + +impl AnalyticsObservationReducer { + /// Ingests an app.used observation and emits the current analytics event. + pub(crate) async fn ingest_app_used( + &mut self, + observation: events::AppUsed<'_>, + out: &mut Vec, + ) { + let tracking = TrackEventsContext { + model_slug: observation.model_slug.to_string(), + thread_id: observation.thread_id.to_string(), + turn_id: observation.turn_id.to_string(), + }; + let app = AppInvocation { + connector_id: observation.connector_id.map(str::to_string), + app_name: observation.app_name.map(str::to_string), + invocation_type: observation.invocation_type.map(map_invocation_type), + }; + + self.legacy + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::AppUsed(AppUsedInput { tracking, app })), + out, + ) + .await; + } +} + +fn map_invocation_type(invocation_type: events::InvocationType) -> facts::InvocationType { + match invocation_type { + events::InvocationType::Explicit => facts::InvocationType::Explicit, + events::InvocationType::Implicit => facts::InvocationType::Implicit, + } +} diff --git a/codex-rs/observability-derive/BUILD.bazel b/codex-rs/observability-derive/BUILD.bazel new file mode 100644 index 0000000000..ee0df71583 --- /dev/null +++ b/codex-rs/observability-derive/BUILD.bazel @@ -0,0 +1,7 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "observability-derive", + crate_name = "codex_observability_derive", + proc_macro = True, +) diff --git a/codex-rs/observability-derive/Cargo.toml b/codex-rs/observability-derive/Cargo.toml new file mode 100644 index 0000000000..1fba8b8485 --- /dev/null +++ b/codex-rs/observability-derive/Cargo.toml @@ -0,0 +1,16 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-observability-derive" +version.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1" +quote = "1" +syn = { version = "2", features = ["full", "extra-traits"] } + +[lints] +workspace = true diff --git a/codex-rs/observability-derive/src/lib.rs b/codex-rs/observability-derive/src/lib.rs new file mode 100644 index 0000000000..0f8412e02d --- /dev/null +++ b/codex-rs/observability-derive/src/lib.rs @@ -0,0 +1,208 @@ +//! Derive macro implementation for `codex-observability`. +//! +//! The macro is intentionally small: it turns annotated struct fields into +//! `ObservationFieldVisitor` calls and refuses to compile if any exported field +//! is missing policy metadata. Filtering, serialization, redaction, and +//! destination-specific mapping stay in sinks/reducers. + +use proc_macro::TokenStream; +use quote::format_ident; +use quote::quote; +use syn::Data; +use syn::DeriveInput; +use syn::Expr; +use syn::Fields; +use syn::LitStr; +use syn::Path; +use syn::parse_macro_input; + +/// Derives `codex_observability::Observation` for a named struct. +/// +/// Required attributes: +/// +/// - `#[observation(name = "domain.event")]` on the struct. +/// - `#[obs(level = "basic|detailed|trace", class = "...")]` on every field. +/// +/// Event definitions inside `codex-observability` itself may use +/// `#[observation(crate = "crate")]` so generated code refers to local types +/// instead of the externally visible crate path. +#[proc_macro_derive(Observation, attributes(observation, obs))] +pub fn derive_observation(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); + expand_observation(input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() +} + +fn expand_observation(input: DeriveInput) -> syn::Result { + let name = input.ident; + let observation_attr = observation_attr(&input.attrs)?; + let event_name = observation_attr.name; + let crate_path = observation_attr.crate_path; + let Data::Struct(data) = input.data else { + return Err(syn::Error::new_spanned( + name, + "Observation can only be derived for structs", + )); + }; + let Fields::Named(fields) = data.fields else { + return Err(syn::Error::new_spanned( + name, + "Observation requires a struct with named fields", + )); + }; + + let mut visits = Vec::new(); + for field in fields.named { + let Some(field_name) = field.ident else { + continue; + }; + let meta = obs_meta(&field.attrs, &field_name, &crate_path)?; + let field_name_lit = LitStr::new(&field_name.to_string(), field_name.span()); + visits.push(quote! { + visitor.field(#field_name_lit, #meta, &self.#field_name); + }); + } + + let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl(); + Ok(quote! { + impl #impl_generics #crate_path::Observation for #name #ty_generics #where_clause { + const NAME: &'static str = #event_name; + + fn visit_fields(&self, visitor: &mut V) { + #(#visits)* + } + } + }) +} + +struct ObservationAttr { + name: LitStr, + crate_path: Path, +} + +fn observation_attr(attrs: &[syn::Attribute]) -> syn::Result { + for attr in attrs { + if !attr.path().is_ident("observation") { + continue; + } + let mut name = None; + let mut crate_path = None; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("name") { + name = Some(meta.value()?.parse::()?); + Ok(()) + } else if meta.path.is_ident("crate") { + let value = meta.value()?.parse::()?; + // The override is needed only when deriving observations from + // inside `codex-observability`, where the external crate name is + // not available. Ordinary users get the stable public path. + crate_path = Some(value.parse::()?); + Ok(()) + } else { + Err(meta.error("unsupported observation attribute")) + } + })?; + if let Some(name) = name { + return Ok(ObservationAttr { + name, + crate_path: crate_path.unwrap_or_else(|| syn::parse_quote!(::codex_observability)), + }); + } + } + Err(syn::Error::new( + proc_macro2::Span::call_site(), + "missing #[observation(name = \"...\")]", + )) +} + +fn obs_meta( + attrs: &[syn::Attribute], + field_name: &syn::Ident, + crate_path: &Path, +) -> syn::Result { + for attr in attrs { + if !attr.path().is_ident("obs") { + continue; + } + let mut level = None; + let mut class = None; + attr.parse_nested_meta(|meta| { + if meta.path.is_ident("level") { + level = Some(meta.value()?.parse::()?); + Ok(()) + } else if meta.path.is_ident("class") { + class = Some(meta.value()?.parse::()?); + Ok(()) + } else { + Err(meta.error("unsupported obs attribute")) + } + })?; + let level = level.ok_or_else(|| { + syn::Error::new_spanned(attr, "missing obs level, expected level = \"...\"") + })?; + let class = class.ok_or_else(|| { + syn::Error::new_spanned(attr, "missing obs class, expected class = \"...\"") + })?; + let detail = detail_expr(&level, crate_path)?; + let data_class = data_class_expr(&class, crate_path)?; + return Ok(quote! { + #crate_path::FieldMeta::new(#detail, #data_class) + }); + } + Err(syn::Error::new_spanned( + field_name, + "missing #[obs(level = \"...\", class = \"...\")]", + )) +} + +fn detail_expr(value: &LitStr, crate_path: &Path) -> syn::Result { + enum_expr( + value, + "detail level", + &[ + ("basic", "Basic"), + ("detailed", "Detailed"), + ("trace", "Trace"), + ], + quote!(#crate_path::DetailLevel), + ) +} + +fn data_class_expr(value: &LitStr, crate_path: &Path) -> syn::Result { + enum_expr( + value, + "data class", + &[ + ("identifier", "Identifier"), + ("operational", "Operational"), + ("environment", "Environment"), + ("content", "Content"), + ("secret_risk", "SecretRisk"), + ], + quote!(#crate_path::DataClass), + ) +} + +fn enum_expr( + value: &LitStr, + label: &str, + variants: &[(&str, &str)], + path: proc_macro2::TokenStream, +) -> syn::Result { + let raw = value.value(); + let Some((_, variant)) = variants.iter().find(|(name, _)| *name == raw) else { + let expected = variants + .iter() + .map(|(name, _)| format!("\"{name}\"")) + .collect::>() + .join(", "); + return Err(syn::Error::new_spanned( + value, + format!("invalid {label} {raw:?}, expected one of {expected}"), + )); + }; + let variant = format_ident!("{variant}"); + let expr: Expr = syn::parse_quote!(#path::#variant); + Ok(quote!(#expr)) +} diff --git a/codex-rs/observability/BUILD.bazel b/codex-rs/observability/BUILD.bazel new file mode 100644 index 0000000000..17ee1c0e3c --- /dev/null +++ b/codex-rs/observability/BUILD.bazel @@ -0,0 +1,6 @@ +load("//:defs.bzl", "codex_rust_crate") + +codex_rust_crate( + name = "observability", + crate_name = "codex_observability", +) diff --git a/codex-rs/observability/Cargo.toml b/codex-rs/observability/Cargo.toml new file mode 100644 index 0000000000..434588716f --- /dev/null +++ b/codex-rs/observability/Cargo.toml @@ -0,0 +1,21 @@ +[package] +edition.workspace = true +license.workspace = true +name = "codex-observability" +version.workspace = true + +[lib] +doctest = true +name = "codex_observability" +path = "src/lib.rs" + +[dependencies] +codex-observability-derive = { workspace = true } +serde = { workspace = true, features = ["derive"] } + +[dev-dependencies] +pretty_assertions = { workspace = true } +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/codex-rs/observability/src/events.rs b/codex-rs/observability/src/events.rs new file mode 100644 index 0000000000..cdcaba2701 --- /dev/null +++ b/codex-rs/observability/src/events.rs @@ -0,0 +1,43 @@ +//! Canonical Codex observation event definitions. + +use crate::Observation; +use serde::Serialize; + +/// How an app/tool/plugin capability was selected by the user or system. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum InvocationType { + /// The user explicitly mentioned or selected the capability. + Explicit, + /// Codex inferred that the capability should be used. + Implicit, +} + +/// Observation emitted when Codex uses an app connector during a turn. +#[derive(Observation)] +#[observation(name = "app.used", crate = "crate")] +pub struct AppUsed<'a> { + /// Model slug active for the turn where the app was used. + #[obs(level = "basic", class = "operational")] + pub model_slug: &'a str, + + /// Thread that owns the turn. + #[obs(level = "basic", class = "identifier")] + pub thread_id: &'a str, + + /// Turn where the app was used. + #[obs(level = "basic", class = "identifier")] + pub turn_id: &'a str, + + /// Stable connector identifier when the app is backed by a connector. + #[obs(level = "basic", class = "identifier")] + pub connector_id: Option<&'a str>, + + /// User-facing app name. + #[obs(level = "basic", class = "operational")] + pub app_name: Option<&'a str>, + + /// Whether usage was explicit or implicit. + #[obs(level = "basic", class = "operational")] + pub invocation_type: Option, +} diff --git a/codex-rs/observability/src/lib.rs b/codex-rs/observability/src/lib.rs new file mode 100644 index 0000000000..bef7314bed --- /dev/null +++ b/codex-rs/observability/src/lib.rs @@ -0,0 +1,127 @@ +//! Shared semantic observation API for Codex runtime facts. +//! +//! Observations describe facts that occurred in Codex. Destination-specific +//! systems such as analytics, rollout trace, OTEL events, and OTEL metrics +//! consume observations through sinks and reducers. +//! +//! Field metadata is required because sinks must decide whether they may read a +//! field before serializing or exporting it. Missing field annotations are a +//! compile-time error: +//! +//! ```compile_fail +//! use codex_observability::Observation; +//! +//! #[derive(Observation)] +//! #[observation(name = "example.missing_field_meta")] +//! struct MissingFieldMeta { +//! field: &'static str, +//! } +//! ``` +//! +//! Observation names are also required: +//! +//! ```compile_fail +//! use codex_observability::Observation; +//! +//! #[derive(Observation)] +//! struct MissingObservationName { +//! #[obs(level = "basic", class = "operational")] +//! field: &'static str, +//! } +//! ``` + +pub mod events; + +pub use codex_observability_derive::Observation; +use serde::Serialize; + +/// A runtime fact emitted by Codex. +/// +/// Implementations visit every exported field together with its field metadata. +/// Sinks use that metadata to apply destination-specific policy before +/// serialization, storage, or export. +pub trait Observation { + /// Stable semantic event name, for example `turn.started`. + const NAME: &'static str; + + /// Visits the fields that make up this observation. + fn visit_fields(&self, visitor: &mut V); +} + +/// Receives observation fields after policy metadata has been attached. +/// +/// Implementations should inspect `meta` before serializing `value`. This keeps +/// remote sinks from accidentally materializing local-only content fields. +pub trait ObservationFieldVisitor { + /// Visits one field from an observation. + fn field(&mut self, name: &'static str, meta: FieldMeta, value: &T); +} + +/// Consumes observations. +/// +/// A sink may serialize immediately, reduce into another event shape, or fan +/// out to additional sinks. The trait is generic so callers can pass borrowed +/// typed observations without allocating an intermediate event object. +pub trait ObservationSink { + /// Observes a single typed event. + fn observe(&self, event: &E); +} + +/// Policy metadata attached to a single observation field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct FieldMeta { + /// How much detail a sink must be allowed to read before consuming the field. + pub detail: DetailLevel, + /// Semantic/privacy class for the field. + pub class: DataClass, +} + +impl FieldMeta { + /// Creates metadata for a field. + pub const fn new(detail: DetailLevel, class: DataClass) -> Self { + Self { detail, class } + } +} + +/// Coarse detail level for a field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DetailLevel { + /// Lifecycle, identifiers, status, counts, model/config, and timing. + Basic, + /// Bounded previews and richer runtime summaries. + Detailed, + /// Raw or near-raw diagnostic evidence intended for local traces. + Trace, +} + +/// Semantic/privacy class for a field. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DataClass { + /// Thread IDs, turn IDs, call IDs, and similar correlation identifiers. + Identifier, + /// Status, duration, model, provider, token counts, and tool kind. + Operational, + /// Working directory, repository, OS/runtime, or client metadata. + Environment, + /// User text, assistant text, command text, tool output, or model payloads. + Content, + /// Headers, environment values, auth-like payloads, or raw request blobs. + SecretRisk, +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + #[test] + fn field_meta_preserves_detail_and_class() { + assert_eq!( + FieldMeta::new(DetailLevel::Trace, DataClass::Content), + FieldMeta { + detail: DetailLevel::Trace, + class: DataClass::Content, + } + ); + } +} diff --git a/codex-rs/observability/tests/derive.rs b/codex-rs/observability/tests/derive.rs new file mode 100644 index 0000000000..201b15433f --- /dev/null +++ b/codex-rs/observability/tests/derive.rs @@ -0,0 +1,81 @@ +use codex_observability::DataClass; +use codex_observability::DetailLevel; +use codex_observability::FieldMeta; +use codex_observability::Observation; +use codex_observability::ObservationFieldVisitor; +use pretty_assertions::assert_eq; +use serde_json::Value; + +#[derive(Observation)] +#[observation(name = "turn.config_resolved")] +struct TurnConfigResolved<'a> { + #[obs(level = "basic", class = "identifier")] + thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + model: &'a str, +} + +#[derive(Debug, PartialEq)] +struct CapturedField { + name: &'static str, + meta: FieldMeta, + value: Value, +} + +#[derive(Default)] +struct CapturingVisitor { + fields: Vec, +} + +impl ObservationFieldVisitor for CapturingVisitor { + fn field( + &mut self, + name: &'static str, + meta: FieldMeta, + value: &T, + ) { + let value = match serde_json::to_value(value) { + Ok(value) => value, + Err(err) => panic!("field should serialize: {err}"), + }; + self.fields.push(CapturedField { name, meta, value }); + } +} + +#[test] +fn derive_visits_annotated_fields_with_metadata() { + let event = TurnConfigResolved { + thread_id: "thread-1", + turn_id: "turn-1", + model: "gpt-5.4", + }; + + let mut visitor = CapturingVisitor::default(); + event.visit_fields(&mut visitor); + + assert_eq!(TurnConfigResolved::NAME, "turn.config_resolved"); + assert_eq!( + visitor.fields, + vec![ + CapturedField { + name: "thread_id", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Identifier), + value: Value::String("thread-1".to_string()), + }, + CapturedField { + name: "turn_id", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Identifier), + value: Value::String("turn-1".to_string()), + }, + CapturedField { + name: "model", + meta: FieldMeta::new(DetailLevel::Basic, DataClass::Operational), + value: Value::String("gpt-5.4".to_string()), + }, + ] + ); +} diff --git a/docs/observability-event-stream-design.md b/docs/observability-event-stream-design.md new file mode 100644 index 0000000000..ab84cb883a --- /dev/null +++ b/docs/observability-event-stream-design.md @@ -0,0 +1,417 @@ +# Observability Event Stream + +Codex currently has three overlapping observability systems: + +- `codex-rs/analytics` reduces runtime facts into remote product analytics. +- `codex-rs/rollout-trace` records rich local diagnostic traces and reduces + them into a replay/debug graph. +- `codex-rs/otel` emits logs, trace-safe events, and metrics, while native + `tracing` spans carry execution scope and W3C trace context. + +The overlap is real: thread lifecycle, turns, model calls, tools, compaction, +subagents, transport, and feature usage are described more than once. Directly +merging the crates would couple incompatible outputs. The shared layer should +instead define the facts Codex emits; each destination should decide how those +facts are filtered, reduced, stored, or exported. + +## Decision + +Introduce a shared observation stream for semantic facts. + +```text +Observations own semantic facts and metrics derived from facts. +tracing owns spans and trace context propagation. +OTEL is a consumer of observations, not a second event taxonomy. +``` + +This gives one rule for callers: + +- Use observations when a thing happened. +- Use sinks/reducers for analytics, rollout trace, OTEL events, OTEL metrics, + local buffers, and feedback upload. +- Use `tracing::span!` when code needs active execution scope, parent/child + causality, span mutation, or W3C context propagation. + +The target state should not be a long-lived mix where some facts use +observations and other facts call analytics or OTEL directly. Direct OTEL code +should remain for exporter setup, native spans, context propagation, and +low-level tracing plumbing. + +## Goals + +- Define canonical observation events that describe what Codex did, not where + the data is going. +- Keep analytics, rollout trace, OTEL logs, and OTEL metrics as projections. +- Apply filtering at field level so a single observation can contain both safe + aggregate fields and richer local-only evidence. +- Preserve existing analytics and OTEL output behavior while moving callsites + toward the shared stream. +- Support local rich retention for feedback/debug upload with explicit policy. +- Prove analytics equivalence with side-by-side conformance tests. + +## Non-Goals + +- Do not make analytics facts the source of truth for rollout trace. +- Do not make rollout trace raw events the source of truth for analytics. +- Do not encode downstream event names such as `codex_turn_event` or + `codex.tool_result` into the canonical taxonomy. +- Do not replace native spans with `SpanStarted` / `SpanEnded` observations. +- Do not require full rollout trace graph equivalence before the first version; + rollout trace is experimental and can use lighter validation initially. + +## Shape + +```text +codex-core / app-server + emit observations + +codex-observability + observation traits, field metadata, sink policy, event definitions + +codex-observability-derive + derive macro for field metadata and visitors + +sinks/reducers + analytics projection -> TrackEventRequest + rollout trace projection -> local trace bundle + reduced graph + OTEL event projection -> current log and trace-safe event names + OTEL metric projection -> current counters and histograms + ringbuffer -> recent rich local observations + +native tracing spans + remain direct tracing instrumentation +``` + +The derive crate exists because Rust does not expose struct field attributes at +runtime. The public API can still re-export the derive from +`codex-observability` so most callsites only import one crate. + +## Why Not Just OTEL? + +An alternative is to emit semantic facts as ordinary `tracing` events and let a +local subscriber, OTEL exporter, analytics reducer, or trace reducer all consume +the same event stream. That is attractive because Codex already uses +`tracing`, and local JSON event capture is easy to wire up. + +We should still avoid making OTEL/tracing events the canonical schema: + +- `tracing` fields are stringly typed at the reducer boundary. Renames or type + changes become runtime reducer failures instead of Rust compile errors. +- OTEL trace-safe export has a different privacy posture than local diagnostic + traces. Rich local fields such as tool output, prompts, model payloads, and + terminal output should not share a target with remotely exported span events. +- Field-level policy needs metadata that `tracing` does not preserve in a + structured way. We need to know both detail level and data class before a + sink serializes a field. +- Analytics conformance is easier from typed observations than from flattened + JSON logs, because reducers can match on Rust event types and fields. +- Spans and semantic facts have different lifecycles. Spans model active scope + and context propagation; observations model facts that happened. + +OTEL should therefore consume observations for semantic events and derived +metrics, while native `tracing` remains the right tool for spans, context +propagation, and exporter plumbing. A local trace sink can still write JSONL +bundles; it should write observation envelopes rather than raw +`tracing_subscriber` event JSON. + +## Observation API + +The shared schema API can be small: + +```rust +pub trait Observation { + const NAME: &'static str; + + fn visit_fields(&self, visitor: &mut V); +} + +pub trait ObservationFieldVisitor { + fn field( + &mut self, + name: &'static str, + meta: FieldMeta, + value: &T, + ); +} + +pub trait ObservationSink { + fn observe(&self, event: &E); +} +``` + +Example: + +```rust +use codex_observability::Observation; + +#[derive(Observation)] +#[observation(name = "turn.config_resolved")] +struct TurnConfigResolved<'a> { + #[obs(level = "basic", class = "identifier")] + thread_id: &'a str, + + #[obs(level = "basic", class = "identifier")] + turn_id: &'a str, + + #[obs(level = "basic", class = "operational")] + model: &'a str, +} +``` + +The derive only exposes metadata. It should not implement analytics mapping, +trace persistence, redaction, schema generation, or export behavior. + +## Callsite API + +Product/runtime code should usually call semantic helper methods rather than +inline-constructing large observation structs. Direct struct construction is +fine in tests, very small leaf events, or helper internals, but it should not be +the normal instrumentation style. + +The helper layer should do the tedious, failure-prone work near the callsite: +extract common IDs, join session/thread/turn context, compute durations, +normalize status and error fields, avoid trace-only allocations when no sink +can read them, and then emit the typed observation. + +Example shape: + +```rust +observability.turn().config_resolved(&session, &turn_context, &input).await; +observability.tool().ended(&invocation, status, &response); +observability.transport().api_request_completed(&request, &outcome); +``` + +This mirrors the useful part of the existing systems: + +- rollout trace has `RolloutTraceRecorder::record_*` helpers +- analytics has `AnalyticsEventsClient::track_*` helpers +- OTEL has `SessionTelemetry::record_*` helpers + +The helper layer must not become a second taxonomy. It is ergonomic glue over +canonical observation structs. Prefer small domain-specific helper groups such +as turn, tool, inference, transport, compaction, and feature usage over one +large recorder with every method in the system. + +## Rich Payloads + +Start simple. Ordinary observation fields should be eager values or borrowed +references. Sinks should reject fields by metadata before serializing them, so +analytics and OTEL metrics do not inspect trace/content fields. + +Some trace fields point at large objects that usually already exist at the +callsite: model requests, model responses, tool invocations, tool results, or +terminal output. Prefer passing borrowed serializable views of those objects to +the observation helper rather than building `serde_json::Value` eagerly. + +Do not introduce a generalized lazy-field system in the first implementation. +Closures such as `|| build_trace_payload(...)` are useful when constructing the +trace view itself is expensive, but they add API complexity. Treat them as a +later optimization for measured hot spots, not as the default observation +style. + +## Field Policy + +Filtering must be field-level. Event-level labels such as basic/detailed/trace +are too coarse because one event often contains both remote-safe counters and +local-only content. + +Each field should carry at least: + +- **Detail level**: `basic`, `detailed`, or `trace`. +- **Data class**: `identifier`, `operational`, `environment`, `content`, or + `secret_risk`. + +Detail level is not privacy by itself. A tiny field can still be unsafe for +remote export, and a trace-level field can be trace-level because it is large +rather than sensitive. Sinks must filter by both axes. + +Expected sink policies: + +- Analytics allows basic identifiers/operational fields plus selected + environment fields; it denies content and secret-risk fields. +- Rollout trace allows rich local fields, with explicit redaction rules for + secret-risk material. +- OTEL logs preserve today's log export behavior, including account/email only + where today's policy allows them. +- OTEL trace-safe events prefer lengths, counts, status, timing, and coarse + categories over content. +- OTEL metrics read only the dimensions needed for existing metric names/tags. +- Feedback upload applies an explicit user-approved policy over the ringbuffer. + +## Event Taxonomy + +The taxonomy should be designed from Codex workflows, not from existing +destination schemas. Names should describe facts that occurred in the system: +`turn.ended`, `tool_call.ended`, `transport.api_request_completed`. + +Rules: + +- Prefer stable domain facts over analytics, OTEL, or trace storage names. +- Model lifecycle facts explicitly when downstream reducers need ordering or + duration. +- Keep event count small until a consumer needs a distinct fact. +- Put richness in fields, not in parallel event variants. +- Add transport/runtime facts when they are first-class telemetry today, even + if analytics ignores them. +- Keep an escape hatch only for local experimental tracing, not remote export. + +Initial workflow coverage should be chosen by conformance need: + +| Workflow | Canonical examples | Primary consumers | +| --- | --- | --- | +| Session/thread | `session.config_resolved`, `thread.started`, `thread.ended` | analytics, OTEL, rollout | +| Turn lifecycle | `turn.requested`, `turn.started`, `turn.ended`, `turn.token_usage_observed` | analytics, OTEL, rollout | +| Turn timing | `turn.first_token_observed`, `turn.first_message_observed` | OTEL metrics, rollout | +| Model I/O | `inference.started`, `inference.sse_event_observed`, `inference.completed`, `inference.failed` | OTEL, rollout | +| Tools | `tool_call.started`, `tool_call.approval_resolved`, `tool_call.ended` | OTEL, rollout | +| Compaction | `compaction.started`, `compaction.installed`, `compaction.ended` | analytics, rollout | +| Agents | `agent.task_sent`, `agent.message_sent`, `agent.result_delivered`, `agent.closed` | rollout, analytics subset | +| Product features | `skill.invoked`, `app.used`, `plugin.state_changed`, `guardian.review_completed` | analytics | +| Transport/auth | `transport.api_request_completed`, `transport.websocket_request_completed`, `auth.recovery_step_completed` | OTEL | + +This table is not a complete schema. Each event still needs a typed Rust +definition with field annotations before implementation. + +## Projections + +### Analytics + +Analytics becomes a reducer over observations. Existing product events remain +output schemas. + +| Observations | Analytics output | +| --- | --- | +| `thread.started` | `codex_thread_initialized` | +| turn config/lifecycle/token observations | `codex_turn_event` | +| `turn.steer_resolved` | `codex_turn_steer_event` | +| compaction lifecycle observations | `codex_compaction_event` | +| skill/app/plugin/guardian observations | existing feature events | + +The analytics reducer continues to own batching, deduplication, connection +metadata joins, and product event naming. + +### Rollout Trace + +Rollout trace consumes observations and decides which fields become trace +payloads. Trace storage details such as `RawPayloadRef` should remain a trace +implementation detail, not part of the shared event definitions. + +Example: `inference.completed` can include response material as a trace-level +field. The trace sink may write it to `payloads/*.json`; analytics and OTEL +metrics never read it. + +### OTEL + +OTEL consumes observations for every semantic fact and derived metric. The +projection preserves existing exported names while removing separate callsite +event definitions. + +| Observations | OTEL output | +| --- | --- | +| session/thread observations | `codex.conversation_starts`, `codex.thread.started` | +| `turn.input_received` | `codex.user_prompt` | +| `tool_call.approval_resolved` | `codex.tool_decision` | +| `tool_call.ended` | `codex.tool_result`, tool count/duration metrics | +| transport observations | API/websocket events and count/duration metrics | +| `auth.recovery_step_completed` | `codex.auth_recovery` | +| inference/SSE observations | `codex.sse_event`, SSE duration, response timing metrics | +| turn timing/token observations | TTFT, TTFM, E2E duration, token usage metrics | + +Log-only and trace-safe OTEL variants can be emitted from the same observation +by applying different field policies. + +## Conformance Testing + +The highest-value equivalence target is analytics because it is existing +product behavior. + +```text +same E2E Codex run + legacy analytics facts -> current AnalyticsReducer -> TrackEventRequest JSON + observations -> analytics observation reducer -> TrackEventRequest JSON + +assert exact equality after stable JSON normalization +``` + +At least one conformance path should apply the analytics field policy so bad +or missing annotations fail tests. Recommended first scenarios: + +- thread start +- normal turn +- failed or interrupted turn +- accepted and rejected turn steering +- compaction +- skill, app, plugin, and guardian events +- subagent thread start + +Rollout trace can start with observation-to-trace unit tests plus one or two +end-to-end smoke tests that write and reduce a bundle. Full graph equivalence +is not necessary for the initial experimental rollout feature. + +OTEL should get projection tests that compare exported event names, fields, +metric names, and tags for representative observations. Native span tests +should remain in the tracing layer. + +## Implementation Stages + +Build the proof of life on a fresh `main` branch, but keep commits separable so +the working branch can later be split into reviewable PRs. +Each stage should leave a small, documented API surface and a passing local +verification gate. Non-essential behavior should stay as a TODO until a later +stage needs it. + +1. **Observation foundation** + - Add `codex-observability` and `codex-observability-derive`. + - Verify with crate-local tests that derived observations expose field + metadata and reject missing annotations at compile time. + - Run: `cargo test -p codex-observability`. + +2. **Analytics conformance slice** + - Add the first typed observations and an analytics projection for one + existing analytics flow. + - Verify legacy analytics facts and observation-derived facts produce + identical `TrackEventRequest` JSON after stable normalization. + - Run: targeted `cargo test -p codex-analytics`. + +3. **E2E analytics shadowing** + - Capture legacy facts and observations from the same core/app-server test + run for selected scenarios. + - Verify exact analytics output equality for thread start, normal turn, + failed/interrupted turn, steering, compaction, and one feature event. + - Run the targeted core/app-server integration tests added for each + scenario. + +4. **Rollout trace projection** + - Add a rollout trace sink that consumes the same observations for + thread/turn, inference, and tool lifecycle. + - Verify with observation-to-trace unit tests and one smoke test that writes + and reduces a local bundle. + +5. **OTEL projection proof** + - Add a small OTEL projection for a high-value event such as tool result. + - Verify exported event names, field policy, metric names, and tags match + the current behavior. + +## Migration + +Start with a shadow observation stream and conformance tests. Once analytics +equivalence is reliable, move callsites from direct analytics facts to +observations. + +Rollout trace can adopt observations directly because it is experimental. + +OTEL should be migrated by category rather than left permanently mixed: +tool/user/turn events, transport events, auth events, and derived metrics can +move behind the OTEL projection. Native spans and trace context stay where they +are. + +## Open Questions + +- What is the smallest useful field-class set? +- Should timestamps and sequence numbers live in the observation envelope or be + supplied by sinks? +- Where should fanout live so callsites do not grow `codex-core` further? +- How should borrowed trace-level payloads avoid hot-path allocations while + still allowing sinks to serialize when policy permits? +- Which direct OTEL counters are first-class observations, and which are + metrics derived from broader observations?