openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-09-04 15:08:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Track history notes thread hint outcomes (#42247)

Browse Source 
## What changed

- Emit a `codex_thread_hint_status` analytics event for each native history-notes thread hint attempt.
- Report whether retrieval was successful or failed along with thread context and timing, without including hint contents.
- Treat valid empty responses as successful retrievals while continuing to omit them from the context window.

## Testing

- Extend the app-server history-notes tests to verify success, empty-result success, and backend failure statuses.

GitOrigin-RevId: b9462f312847e8c871ed2be0c8cf8df0928a7fdd
This commit is contained in:
pmccrary-oai
2026-09-02 09:01:38 +00:00
committed by copyberry
parent eb10d91e48
commit bdfd769640
10 changed files with 118 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
codex-rs/Cargo.lock generated
View File

@@ -3425,6 +3425,7 @@ dependencies = [
name = "codex-history-notes-extension"
version = "0.0.0"
dependencies = [
"codex-analytics",
"codex-api",
"codex-client",
"codex-config",

6
codex-rs/analytics/src/client.rs
View File

@@ -505,6 +505,12 @@ impl AnalyticsEventsClient {
))));
}
pub fn track_thread_hint_status(&self, event: crate::thread_hint::ThreadHintStatusEvent) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::ThreadHintStatus(Box::new(event)),
));
}
pub fn track_image_preparation(&self, fact: ImagePreparationFact) {
self.record_fact(AnalyticsFact::Custom(
CustomAnalyticsFact::ImagePreparation(Box::new(fact)),

1
codex-rs/analytics/src/events.rs
View File

@@ -76,6 +76,7 @@ pub(crate) enum TrackEventRequest {
HookRun(CodexHookRunEventRequest),
Compaction(Box<CodexCompactionEventRequest>),
Goal(Box<CodexGoalEventRequest>),
ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEventRequest>),
TurnEvent(Box<CodexTurnEventRequest>),
TurnSteer(CodexTurnSteerEventRequest),
ArtifactOperation(CodexArtifactOperationEventRequest),

1
codex-rs/analytics/src/facts.rs
View File

@@ -566,6 +566,7 @@ pub(crate) enum CustomAnalyticsFact {
SubAgentThreadStarted(SubAgentThreadStartedInput),
Compaction(Box<CodexCompactionEvent>),
Goal(Box<CodexGoalEvent>),
ThreadHintStatus(Box<crate::thread_hint::ThreadHintStatusEvent>),
GuardianReview(Box<GuardianReviewEventParams>),
GuardianV2(Box<GuardianV2Event>),
TurnResolvedConfig(Box<TurnResolvedConfigFact>),

3
codex-rs/analytics/src/lib.rs
View File

@@ -6,6 +6,7 @@ mod events;
mod facts;
mod guardian_v2;
mod reducer;
mod thread_hint;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
@@ -74,6 +75,8 @@ pub use facts::TurnTokenUsageFact;
pub use facts::build_track_events_context;
pub use guardian_v2::GuardianV2Event;
pub use guardian_v2::GuardianV2EventKind;
pub use thread_hint::ThreadHintStatus;
pub use thread_hint::ThreadHintStatusEvent;
#[cfg(test)]
mod analytics_client_tests;

28
codex-rs/analytics/src/reducer.rs
View File

@@ -621,6 +621,34 @@ impl AnalyticsReducer {
CustomAnalyticsFact::Goal(input) => {
self.ingest_goal(*input, out);
}
CustomAnalyticsFact::ThreadHintStatus(input) => {
if let Some((connection, thread, metadata)) =
self.thread_context_or_warn(AnalyticsDropSite {
event_name: "codex_thread_hint_status",
thread_id: &input.thread_id,
turn_id: None,
review_id: None,
item_id: None,
})
{
out.push(TrackEventRequest::ThreadHintStatus(Box::new(
crate::thread_hint::ThreadHintStatusEventRequest {
event_type: "codex_thread_hint_status",
event_params: crate::thread_hint::ThreadHintStatusEventParams {
thread_id: input.thread_id,
session_id: metadata.session_id.clone(),
app_server_client: thread.app_server_client(connection),
runtime: connection.runtime.clone(),
thread_source: metadata.thread_source.clone(),
subagent_source: metadata.subagent_source.clone(),
parent_thread_id: metadata.parent_thread_id.clone(),
status: input.status,
occurred_at_ms: input.occurred_at_ms,
},
},
)));
}
}
CustomAnalyticsFact::GuardianV2(input) => {
let event_type = match &input.kind {
GuardianV2EventKind::Classification { .. } => {

38
codex-rs/analytics/src/thread_hint.rs Normal file
View File

@@ -0,0 +1,38 @@
//! Per-attempt thread hint status analytics without hint contents.
use crate::events::CodexAppServerClientMetadata;
use crate::events::CodexRuntimeMetadata;
use codex_protocol::protocol::ThreadSource;
use serde::Serialize;
#[derive(Clone, Copy, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ThreadHintStatus {
Succeeded,
Failed,
}
pub struct ThreadHintStatusEvent {
pub thread_id: String,
pub status: ThreadHintStatus,
pub occurred_at_ms: u64,
}
#[derive(Serialize)]
pub(crate) struct ThreadHintStatusEventRequest {
pub(crate) event_type: &'static str,
pub(crate) event_params: ThreadHintStatusEventParams,
}
#[derive(Serialize)]
pub(crate) struct ThreadHintStatusEventParams {
pub(crate) thread_id: String,
pub(crate) session_id: String,
pub(crate) app_server_client: CodexAppServerClientMetadata,
pub(crate) runtime: CodexRuntimeMetadata,
pub(crate) thread_source: Option<ThreadSource>,
pub(crate) subagent_source: Option<String>,
pub(crate) parent_thread_id: Option<String>,
pub(crate) status: ThreadHintStatus,
pub(crate) occurred_at_ms: u64,
}

28
codex-rs/app-server/tests/suite/v2/history_notes_extension.rs
View File

@@ -1,14 +1,11 @@
use std::time::Duration;
use anyhow::Result;
use app_test_support::ChatGptAuthFixture;
use app_test_support::MockResponsesConfig;
use app_test_support::TestAppServer;
use app_test_support::write_chatgpt_auth;
use codex_app_server_protocol::ThreadStartParams;
use codex_app_server_protocol::TurnStartParams;
use codex_app_server_protocol::UserInput;
use codex_config::types::AuthCredentialsStoreMode;
use core_test_support::load_default_config_for_test;
use core_test_support::responses;
use pretty_assertions::assert_eq;
@@ -116,6 +113,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
let codex_home = TempDir::new()?;
MockResponsesConfig::new(&server.uri())
.with_root_config(&format!("chatgpt_base_url = \"{}\"", server.uri()))
.with_model_provider("openai-custom")
.with_provider_name("OpenAI")
.with_provider_base_url(&format!("{}/backend-api/codex", server.uri()))
@@ -125,11 +123,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
server.uri(),
))
.write(codex_home.path())?;
write_chatgpt_auth(
codex_home.path(),
ChatGptAuthFixture::new("access-chatgpt"),
AuthCredentialsStoreMode::File,
)?;
mount_analytics_capture(&server, codex_home.path()).await?;
let mut app_server = TestAppServer::builder()
.with_codex_home(codex_home.path())
@@ -143,7 +137,7 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
timeout(
Duration::from_secs(10),
app_server.start_turn_and_wait_for_completion(TurnStartParams {
thread_id: thread.id,
thread_id: thread.id.clone(),
input: vec![UserInput::Text {
text: "inspect history and notes".to_string(),
text_elements: Vec::new(),
@@ -218,6 +212,22 @@ async fn app_server_uses_configured_notes_backend_for_context_window_hints(
&& item["name"] == "thread_hint"
}));
if use_history_notes_extension {
let event = wait_for_matching_analytics_event(&server, DEFAULT_READ_TIMEOUT, |event| {
event["event_type"] == "codex_thread_hint_status"
&& event["event_params"]["thread_id"] == thread.id
})
.await?;
assert_eq!(
event["event_params"]["status"],
if hint_status == 200 {
"succeeded"
} else {
"failed"
},
);
}
Ok(())
}

1
codex-rs/ext/history-notes/Cargo.toml
View File

@@ -13,6 +13,7 @@ doctest = false
workspace = true
[dependencies]
codex-analytics = { workspace = true }
codex-api = { workspace = true }
codex-client = { workspace = true }
codex-core = { workspace = true }

21
codex-rs/ext/history-notes/src/extension.rs
View File

@@ -1,5 +1,8 @@
use std::sync::Arc;
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::ThreadHintStatus;
use codex_analytics::ThreadHintStatusEvent;
use codex_core::config::Config;
use codex_extension_api::ConfigContributor;
use codex_extension_api::ContentItemKind;
@@ -104,6 +107,15 @@ impl ContextContributor for HistoryNotesExtension {
let Some(identity) = thread_store.get::<HistoryNotesAgentIdentity>() else {
return Vec::new();
};
let track_status = |status| {
if let Some(analytics) = session_store.get::<AnalyticsEventsClient>() {
analytics.track_thread_hint_status(ThreadHintStatusEvent {
thread_id: thread_store.level_id().to_string(),
status,
occurred_at_ms: codex_analytics::now_unix_millis(),
});
}
};
let Ok(result) = config
.backend
.call(
@@ -115,12 +127,19 @@ impl ContextContributor for HistoryNotesExtension {
)
.await
else {
track_status(ThreadHintStatus::Failed);
return Vec::new();
};
let Some(text) = result.get("text").and_then(serde_json::Value::as_str) else {
track_status(ThreadHintStatus::Failed);
return Vec::new();
};
if text.is_empty() || text.len() > MAX_THREAD_HINT_BYTES {
if text.len() > MAX_THREAD_HINT_BYTES {
track_status(ThreadHintStatus::Failed);
return Vec::new();
}
track_status(ThreadHintStatus::Succeeded);
if text.is_empty() {
return Vec::new();
}
vec![PromptFragment::new(