From bdfd769640927a627dd48ab3fcc1ae8bc08bdd0a Mon Sep 17 00:00:00 2001 From: pmccrary-oai Date: Wed, 2 Sep 2026 09:01:38 +0000 Subject: [PATCH] Track history notes thread hint outcomes (#42247) ## What changed - Emit a `codex_thread_hint_status` analytics event for each native history-notes thread hint attempt. - Report whether retrieval was successful or failed along with thread context and timing, without including hint contents. - Treat valid empty responses as successful retrievals while continuing to omit them from the context window. ## Testing - Extend the app-server history-notes tests to verify success, empty-result success, and backend failure statuses. GitOrigin-RevId: b9462f312847e8c871ed2be0c8cf8df0928a7fdd --- codex-rs/Cargo.lock | 1 + codex-rs/analytics/src/client.rs | 6 +++ codex-rs/analytics/src/events.rs | 1 + codex-rs/analytics/src/facts.rs | 1 + codex-rs/analytics/src/lib.rs | 3 ++ codex-rs/analytics/src/reducer.rs | 28 ++++++++++++++ codex-rs/analytics/src/thread_hint.rs | 38 +++++++++++++++++++ .../tests/suite/v2/history_notes_extension.rs | 28 +++++++++----- codex-rs/ext/history-notes/Cargo.toml | 1 + codex-rs/ext/history-notes/src/extension.rs | 21 +++++++++- 10 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 codex-rs/analytics/src/thread_hint.rs diff --git a/codex-rs/Cargo.lock b/codex-rs/Cargo.lock index f9c9089a83..5c3da721a4 100644 --- a/codex-rs/Cargo.lock +++ b/codex-rs/Cargo.lock @@ -3425,6 +3425,7 @@ dependencies = [ name = "codex-history-notes-extension" version = "0.0.0" dependencies = [ + "codex-analytics", "codex-api", "codex-client", "codex-config", diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index 96dce42750..6895825c87 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -505,6 +505,12 @@ impl AnalyticsEventsClient { )))); } + pub fn track_thread_hint_status(&self, event: crate::thread_hint::ThreadHintStatusEvent) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::ThreadHintStatus(Box::new(event)), + )); + } + pub fn track_image_preparation(&self, fact: ImagePreparationFact) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::ImagePreparation(Box::new(fact)), diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 0342ea79c9..a984f2791f 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -76,6 +76,7 @@ pub(crate) enum TrackEventRequest { HookRun(CodexHookRunEventRequest), Compaction(Box), Goal(Box), + ThreadHintStatus(Box), TurnEvent(Box), TurnSteer(CodexTurnSteerEventRequest), ArtifactOperation(CodexArtifactOperationEventRequest), diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 4fb8bbca3d..393429aa54 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -566,6 +566,7 @@ pub(crate) enum CustomAnalyticsFact { SubAgentThreadStarted(SubAgentThreadStartedInput), Compaction(Box), Goal(Box), + ThreadHintStatus(Box), GuardianReview(Box), GuardianV2(Box), TurnResolvedConfig(Box), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 7d2f86b979..1897ec0756 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -6,6 +6,7 @@ mod events; mod facts; mod guardian_v2; mod reducer; +mod thread_hint; use std::time::SystemTime; use std::time::UNIX_EPOCH; @@ -74,6 +75,8 @@ pub use facts::TurnTokenUsageFact; pub use facts::build_track_events_context; pub use guardian_v2::GuardianV2Event; pub use guardian_v2::GuardianV2EventKind; +pub use thread_hint::ThreadHintStatus; +pub use thread_hint::ThreadHintStatusEvent; #[cfg(test)] mod analytics_client_tests; diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index be5b8f24cb..0378f2b19c 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -621,6 +621,34 @@ impl AnalyticsReducer { CustomAnalyticsFact::Goal(input) => { self.ingest_goal(*input, out); } + CustomAnalyticsFact::ThreadHintStatus(input) => { + if let Some((connection, thread, metadata)) = + self.thread_context_or_warn(AnalyticsDropSite { + event_name: "codex_thread_hint_status", + thread_id: &input.thread_id, + turn_id: None, + review_id: None, + item_id: None, + }) + { + out.push(TrackEventRequest::ThreadHintStatus(Box::new( + crate::thread_hint::ThreadHintStatusEventRequest { + event_type: "codex_thread_hint_status", + event_params: crate::thread_hint::ThreadHintStatusEventParams { + thread_id: input.thread_id, + session_id: metadata.session_id.clone(), + app_server_client: thread.app_server_client(connection), + runtime: connection.runtime.clone(), + thread_source: metadata.thread_source.clone(), + subagent_source: metadata.subagent_source.clone(), + parent_thread_id: metadata.parent_thread_id.clone(), + status: input.status, + occurred_at_ms: input.occurred_at_ms, + }, + }, + ))); + } + } CustomAnalyticsFact::GuardianV2(input) => { let event_type = match &input.kind { GuardianV2EventKind::Classification { .. } => { diff --git a/codex-rs/analytics/src/thread_hint.rs b/codex-rs/analytics/src/thread_hint.rs new file mode 100644 index 0000000000..c863558b7c --- /dev/null +++ b/codex-rs/analytics/src/thread_hint.rs @@ -0,0 +1,38 @@ +//! Per-attempt thread hint status analytics without hint contents. + +use crate::events::CodexAppServerClientMetadata; +use crate::events::CodexRuntimeMetadata; +use codex_protocol::protocol::ThreadSource; +use serde::Serialize; + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ThreadHintStatus { + Succeeded, + Failed, +} + +pub struct ThreadHintStatusEvent { + pub thread_id: String, + pub status: ThreadHintStatus, + pub occurred_at_ms: u64, +} + +#[derive(Serialize)] +pub(crate) struct ThreadHintStatusEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: ThreadHintStatusEventParams, +} + +#[derive(Serialize)] +pub(crate) struct ThreadHintStatusEventParams { + pub(crate) thread_id: String, + pub(crate) session_id: String, + pub(crate) app_server_client: CodexAppServerClientMetadata, + pub(crate) runtime: CodexRuntimeMetadata, + pub(crate) thread_source: Option, + pub(crate) subagent_source: Option, + pub(crate) parent_thread_id: Option, + pub(crate) status: ThreadHintStatus, + pub(crate) occurred_at_ms: u64, +} diff --git a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs index d49f66dd6f..8d851ddeef 100644 --- a/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs +++ b/codex-rs/app-server/tests/suite/v2/history_notes_extension.rs @@ -1,14 +1,11 @@ use std::time::Duration; use anyhow::Result; -use app_test_support::ChatGptAuthFixture; use app_test_support::MockResponsesConfig; use app_test_support::TestAppServer; -use app_test_support::write_chatgpt_auth; use codex_app_server_protocol::ThreadStartParams; use codex_app_server_protocol::TurnStartParams; use codex_app_server_protocol::UserInput; -use codex_config::types::AuthCredentialsStoreMode; use core_test_support::load_default_config_for_test; use core_test_support::responses; use pretty_assertions::assert_eq; @@ -116,6 +113,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints( let codex_home = TempDir::new()?; MockResponsesConfig::new(&server.uri()) + .with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri())) .with_model_provider("openai-custom") .with_provider_name("OpenAI") .with_provider_base_url(&format!("{}/backend-api/codex", server.uri())) @@ -125,11 +123,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints( server.uri(), )) .write(codex_home.path())?; - write_chatgpt_auth( - codex_home.path(), - ChatGptAuthFixture::new("access-chatgpt"), - AuthCredentialsStoreMode::File, - )?; + mount_analytics_capture(&server, codex_home.path()).await?; let mut app_server = TestAppServer::builder() .with_codex_home(codex_home.path()) @@ -143,7 +137,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints( timeout( Duration::from_secs(10), app_server.start_turn_and_wait_for_completion(TurnStartParams { - thread_id: thread.id, + thread_id: thread.id.clone(), input: vec![UserInput::Text { text: "inspect history and notes".to_string(), text_elements: Vec::new(), @@ -218,6 +212,22 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints( && item["name"] == "thread_hint" })); + if use_history_notes_extension { + let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| { + event["event_type"] == "codex_thread_hint_status" + && event["event_params"]["thread_id"] == thread.id + }) + .await?; + assert_eq!( + event["event_params"]["status"], + if hint_status == 200 { + "succeeded" + } else { + "failed" + }, + ); + } + Ok(()) } diff --git a/codex-rs/ext/history-notes/Cargo.toml b/codex-rs/ext/history-notes/Cargo.toml index d043ee06e1..82a1922d15 100644 --- a/codex-rs/ext/history-notes/Cargo.toml +++ b/codex-rs/ext/history-notes/Cargo.toml @@ -13,6 +13,7 @@ doctest = false workspace = true [dependencies] +codex-analytics = { workspace = true } codex-api = { workspace = true } codex-client = { workspace = true } codex-core = { workspace = true } diff --git a/codex-rs/ext/history-notes/src/extension.rs b/codex-rs/ext/history-notes/src/extension.rs index 4d3e4e24f2..dfbff12cac 100644 --- a/codex-rs/ext/history-notes/src/extension.rs +++ b/codex-rs/ext/history-notes/src/extension.rs @@ -1,5 +1,8 @@ use std::sync::Arc; +use codex_analytics::AnalyticsEventsClient; +use codex_analytics::ThreadHintStatus; +use codex_analytics::ThreadHintStatusEvent; use codex_core::config::Config; use codex_extension_api::ConfigContributor; use codex_extension_api::ContentItemKind; @@ -104,6 +107,15 @@ impl ContextContributor for HistoryNotesExtension { let Some(identity) = thread_store.get::() else { return Vec::new(); }; + let track_status = |status| { + if let Some(analytics) = session_store.get::() { + analytics.track_thread_hint_status(ThreadHintStatusEvent { + thread_id: thread_store.level_id().to_string(), + status, + occurred_at_ms: codex_analytics::now_unix_millis(), + }); + } + }; let Ok(result) = config .backend .call( @@ -115,12 +127,19 @@ impl ContextContributor for HistoryNotesExtension { ) .await else { + track_status(ThreadHintStatus::Failed); return Vec::new(); }; let Some(text) = result.get("text").and_then(serde_json::Value::as_str) else { + track_status(ThreadHintStatus::Failed); return Vec::new(); }; - if text.is_empty() || text.len() > MAX_THREAD_HINT_BYTES { + if text.len() > MAX_THREAD_HINT_BYTES { + track_status(ThreadHintStatus::Failed); + return Vec::new(); + } + track_status(ThreadHintStatus::Succeeded); + if text.is_empty() { return Vec::new(); } vec![PromptFragment::new(