From bd8a3ac0f9012d0b0bbcc42f7d17a0a82cb7ef07 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Thu, 2 Apr 2026 16:14:41 -0700 Subject: [PATCH 1/3] [codex-analytics] add protocol-native turn timestamps --- codex-rs/app-server-client/src/lib.rs | 4 +++ .../schema/json/ServerNotification.json | 20 +++++++++++ .../codex_app_server_protocol.schemas.json | 20 +++++++++++ .../codex_app_server_protocol.v2.schemas.json | 20 +++++++++++ .../json/v2/TurnCompletedNotification.json | 14 ++++++++ .../json/v2/TurnStartedNotification.json | 6 ++++ .../v2/TurnCompletedNotification.ts | 10 +++++- .../typescript/v2/TurnStartedNotification.ts | 6 +++- .../app-server-protocol/src/protocol/v2.rs | 9 +++++ .../app-server/src/bespoke_event_handling.rs | 36 +++++++++++++++++-- codex-rs/app-server/src/in_process.rs | 2 ++ codex-rs/app-server/src/thread_state.rs | 1 + ...event_processor_with_human_output_tests.rs | 10 ++++++ ...event_processor_with_jsonl_output_tests.rs | 2 ++ codex-rs/exec/src/lib.rs | 1 + .../tests/event_processor_with_json_output.rs | 21 +++++++++++ codex-rs/tui/src/app.rs | 3 ++ codex-rs/tui/src/app/app_server_adapter.rs | 6 ++++ .../tui/src/app/pending_interactive_replay.rs | 2 ++ codex-rs/tui/src/chatwidget.rs | 2 ++ .../tui/src/chatwidget/tests/app_server.rs | 8 +++++ .../src/chatwidget/tests/history_replay.rs | 2 ++ .../tui/src/chatwidget/tests/plan_mode.rs | 3 ++ 23 files changed, 204 insertions(+), 4 deletions(-) diff --git a/codex-rs/app-server-client/src/lib.rs b/codex-rs/app-server-client/src/lib.rs index 39768820de..7ff1322d72 100644 --- a/codex-rs/app-server-client/src/lib.rs +++ b/codex-rs/app-server-client/src/lib.rs @@ -1061,6 +1061,8 @@ mod tests { status: codex_app_server_protocol::TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }) } @@ -1835,6 +1837,8 @@ mod tests { status: codex_app_server_protocol::TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, } ) ) diff --git a/codex-rs/app-server-protocol/schema/json/ServerNotification.json b/codex-rs/app-server-protocol/schema/json/ServerNotification.json index b8b539a03b..251327a560 100644 --- a/codex-rs/app-server-protocol/schema/json/ServerNotification.json +++ b/codex-rs/app-server-protocol/schema/json/ServerNotification.json @@ -3566,6 +3566,19 @@ }, "TurnCompletedNotification": { "properties": { + "completedAt": { + "description": "Unix timestamp (in seconds) when the turn completed.", + "format": "int64", + "type": "integer" + }, + "durationMs": { + "description": "Duration between turn start and completion in milliseconds, if known.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "threadId": { "type": "string" }, @@ -3574,6 +3587,7 @@ } }, "required": [ + "completedAt", "threadId", "turn" ], @@ -3680,6 +3694,11 @@ }, "TurnStartedNotification": { "properties": { + "createdAt": { + "description": "Unix timestamp (in seconds) when the turn started.", + "format": "int64", + "type": "integer" + }, "threadId": { "type": "string" }, @@ -3688,6 +3707,7 @@ } }, "required": [ + "createdAt", "threadId", "turn" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json index a589903032..acc274aaff 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.schemas.json @@ -14364,6 +14364,19 @@ "TurnCompletedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "completedAt": { + "description": "Unix timestamp (in seconds) when the turn completed.", + "format": "int64", + "type": "integer" + }, + "durationMs": { + "description": "Duration between turn start and completion in milliseconds, if known.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "threadId": { "type": "string" }, @@ -14372,6 +14385,7 @@ } }, "required": [ + "completedAt", "threadId", "turn" ], @@ -14640,6 +14654,11 @@ "TurnStartedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "createdAt": { + "description": "Unix timestamp (in seconds) when the turn started.", + "format": "int64", + "type": "integer" + }, "threadId": { "type": "string" }, @@ -14648,6 +14667,7 @@ } }, "required": [ + "createdAt", "threadId", "turn" ], diff --git a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json index f041f8aae8..2421630d43 100644 --- a/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json +++ b/codex-rs/app-server-protocol/schema/json/codex_app_server_protocol.v2.schemas.json @@ -12219,6 +12219,19 @@ "TurnCompletedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "completedAt": { + "description": "Unix timestamp (in seconds) when the turn completed.", + "format": "int64", + "type": "integer" + }, + "durationMs": { + "description": "Duration between turn start and completion in milliseconds, if known.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "threadId": { "type": "string" }, @@ -12227,6 +12240,7 @@ } }, "required": [ + "completedAt", "threadId", "turn" ], @@ -12495,6 +12509,11 @@ "TurnStartedNotification": { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { + "createdAt": { + "description": "Unix timestamp (in seconds) when the turn started.", + "format": "int64", + "type": "integer" + }, "threadId": { "type": "string" }, @@ -12503,6 +12522,7 @@ } }, "required": [ + "createdAt", "threadId", "turn" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json index 770cc920cf..3780541077 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnCompletedNotification.json @@ -1559,6 +1559,19 @@ } }, "properties": { + "completedAt": { + "description": "Unix timestamp (in seconds) when the turn completed.", + "format": "int64", + "type": "integer" + }, + "durationMs": { + "description": "Duration between turn start and completion in milliseconds, if known.", + "format": "int64", + "type": [ + "integer", + "null" + ] + }, "threadId": { "type": "string" }, @@ -1567,6 +1580,7 @@ } }, "required": [ + "completedAt", "threadId", "turn" ], diff --git a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json index 761ddc9a62..94af685a0b 100644 --- a/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json +++ b/codex-rs/app-server-protocol/schema/json/v2/TurnStartedNotification.json @@ -1559,6 +1559,11 @@ } }, "properties": { + "createdAt": { + "description": "Unix timestamp (in seconds) when the turn started.", + "format": "int64", + "type": "integer" + }, "threadId": { "type": "string" }, @@ -1567,6 +1572,7 @@ } }, "required": [ + "createdAt", "threadId", "turn" ], diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/TurnCompletedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/TurnCompletedNotification.ts index e1b151bfa7..66b05f3514 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/TurnCompletedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/TurnCompletedNotification.ts @@ -3,4 +3,12 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Turn } from "./Turn"; -export type TurnCompletedNotification = { threadId: string, turn: Turn, }; +export type TurnCompletedNotification = { threadId: string, turn: Turn, +/** + * Unix timestamp (in seconds) when the turn completed. + */ +completedAt: number, +/** + * Duration between turn start and completion in milliseconds, if known. + */ +durationMs: number | null, }; \ No newline at end of file diff --git a/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartedNotification.ts b/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartedNotification.ts index 34f71b2465..c8ddd257e3 100644 --- a/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartedNotification.ts +++ b/codex-rs/app-server-protocol/schema/typescript/v2/TurnStartedNotification.ts @@ -3,4 +3,8 @@ // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually. import type { Turn } from "./Turn"; -export type TurnStartedNotification = { threadId: string, turn: Turn, }; +export type TurnStartedNotification = { threadId: string, turn: Turn, +/** + * Unix timestamp (in seconds) when the turn started. + */ +createdAt: number, }; \ No newline at end of file diff --git a/codex-rs/app-server-protocol/src/protocol/v2.rs b/codex-rs/app-server-protocol/src/protocol/v2.rs index 82b568515c..538293f1f5 100644 --- a/codex-rs/app-server-protocol/src/protocol/v2.rs +++ b/codex-rs/app-server-protocol/src/protocol/v2.rs @@ -5043,6 +5043,9 @@ pub struct ThreadNameUpdatedNotification { pub struct TurnStartedNotification { pub thread_id: String, pub turn: Turn, + /// Unix timestamp (in seconds) when the turn started. + #[ts(type = "number")] + pub created_at: i64, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] @@ -5069,6 +5072,12 @@ pub struct Usage { pub struct TurnCompletedNotification { pub thread_id: String, pub turn: Turn, + /// Unix timestamp (in seconds) when the turn completed. + #[ts(type = "number")] + pub completed_at: i64, + /// Duration between turn start and completion in milliseconds, if known. + #[ts(type = "number | null")] + pub duration_ms: Option, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)] diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 78c60b0aee..6e288da81e 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -141,6 +141,8 @@ use std::convert::TryFrom; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; use tokio::sync::Mutex; use tokio::sync::oneshot; use tracing::error; @@ -289,6 +291,7 @@ pub(crate) async fn apply_bespoke_event_handling( let notification = TurnStartedNotification { thread_id: conversation_id.to_string(), turn, + created_at: set_turn_started_at(&thread_state).await, }; outgoing .send_server_notification(ServerNotification::TurnStarted(notification)) @@ -1947,8 +1950,10 @@ async fn emit_turn_completed_with_status( event_turn_id: String, status: TurnStatus, error: Option, + started_at_ms: Option, outgoing: &ThreadScopedOutgoingMessageSender, ) { + let completed_at_ms = now_unix_timestamp_millis(); let notification = TurnCompletedNotification { thread_id: conversation_id.to_string(), turn: Turn { @@ -1957,6 +1962,9 @@ async fn emit_turn_completed_with_status( error, status, }, + completed_at: completed_at_ms / 1000, + duration_ms: started_at_ms + .map(|started_at_ms| completed_at_ms.saturating_sub(started_at_ms)), }; outgoing .send_server_notification(ServerNotification::TurnCompleted(notification)) @@ -2105,13 +2113,22 @@ async fn handle_turn_complete( thread_state: &Arc>, ) { let turn_summary = find_and_remove_turn_summary(conversation_id, thread_state).await; + let started_at_ms = turn_summary.started_at_ms; let (status, error) = match turn_summary.last_error { Some(error) => (TurnStatus::Failed, Some(error)), None => (TurnStatus::Completed, None), }; - emit_turn_completed_with_status(conversation_id, event_turn_id, status, error, outgoing).await; + emit_turn_completed_with_status( + conversation_id, + event_turn_id, + status, + error, + started_at_ms, + outgoing, + ) + .await; } async fn handle_turn_interrupted( @@ -2120,18 +2137,33 @@ async fn handle_turn_interrupted( outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { - find_and_remove_turn_summary(conversation_id, thread_state).await; + let turn_summary = find_and_remove_turn_summary(conversation_id, thread_state).await; emit_turn_completed_with_status( conversation_id, event_turn_id, TurnStatus::Interrupted, /*error*/ None, + turn_summary.started_at_ms, outgoing, ) .await; } +async fn set_turn_started_at(thread_state: &Arc>) -> i64 { + let started_at_ms = now_unix_timestamp_millis(); + let mut state = thread_state.lock().await; + state.turn_summary.started_at_ms = Some(started_at_ms); + started_at_ms / 1000 +} + +fn now_unix_timestamp_millis() -> i64 { + let duration = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) +} + async fn handle_thread_rollback_failed( _conversation_id: ThreadId, message: String, diff --git a/codex-rs/app-server/src/in_process.rs b/codex-rs/app-server/src/in_process.rs index eda2c75da8..c2d535a126 100644 --- a/codex-rs/app-server/src/in_process.rs +++ b/codex-rs/app-server/src/in_process.rs @@ -824,6 +824,8 @@ mod tests { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }) )); } diff --git a/codex-rs/app-server/src/thread_state.rs b/codex-rs/app-server/src/thread_state.rs index be5478dd51..e7af824009 100644 --- a/codex-rs/app-server/src/thread_state.rs +++ b/codex-rs/app-server/src/thread_state.rs @@ -44,6 +44,7 @@ pub(crate) enum ThreadListenerCommand { /// Per-conversation accumulation of the latest states e.g. error message while a turn runs. #[derive(Default, Clone)] pub(crate) struct TurnSummary { + pub(crate) started_at_ms: Option, pub(crate) file_change_started: HashSet, pub(crate) command_execution_started: HashSet, pub(crate) last_error: Option, diff --git a/codex-rs/exec/src/event_processor_with_human_output_tests.rs b/codex-rs/exec/src/event_processor_with_human_output_tests.rs index 2b625dd564..18d08f3bcd 100644 --- a/codex-rs/exec/src/event_processor_with_human_output_tests.rs +++ b/codex-rs/exec/src/event_processor_with_human_output_tests.rs @@ -157,6 +157,8 @@ fn turn_completed_recovers_final_message_from_turn_items() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: Turn { id: "turn-1".to_string(), items: vec![ThreadItem::AgentMessage { @@ -201,6 +203,8 @@ fn turn_completed_overwrites_stale_final_message_from_turn_items() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: Turn { id: "turn-1".to_string(), items: vec![ThreadItem::AgentMessage { @@ -246,6 +250,8 @@ fn turn_completed_preserves_streamed_final_message_when_turn_items_are_empty() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: Turn { id: "turn-1".to_string(), items: Vec::new(), @@ -286,6 +292,8 @@ fn turn_failed_clears_stale_final_message() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: Turn { id: "turn-1".to_string(), items: Vec::new(), @@ -327,6 +335,8 @@ fn turn_interrupted_clears_stale_final_message() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: Turn { id: "turn-1".to_string(), items: Vec::new(), diff --git a/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs b/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs index ffb4d1ed01..1f8a00b579 100644 --- a/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs +++ b/codex-rs/exec/src/event_processor_with_jsonl_output_tests.rs @@ -29,6 +29,8 @@ fn failed_turn_does_not_overwrite_output_last_message_file() { let status = processor.process_server_notification(ServerNotification::TurnCompleted( codex_app_server_protocol::TurnCompletedNotification { thread_id: "thread-1".to_string(), + completed_at: 0, + duration_ms: None, turn: codex_app_server_protocol::Turn { id: "turn-1".to_string(), items: Vec::new(), diff --git a/codex-rs/exec/src/lib.rs b/codex-rs/exec/src/lib.rs index 5a396a9810..5609fd0344 100644 --- a/codex-rs/exec/src/lib.rs +++ b/codex-rs/exec/src/lib.rs @@ -717,6 +717,7 @@ async fn run_exec_session(args: ExecRunArgs) -> anyhow::Result<()> { TurnStartedNotification { thread_id: response.review_thread_id.clone(), turn: response.turn.clone(), + created_at: 0, }, )); let task_id = response.turn.id; diff --git a/codex-rs/exec/tests/event_processor_with_json_output.rs b/codex-rs/exec/tests/event_processor_with_json_output.rs index 5491e895e2..ed28aa8c85 100644 --- a/codex-rs/exec/tests/event_processor_with_json_output.rs +++ b/codex-rs/exec/tests/event_processor_with_json_output.rs @@ -145,6 +145,7 @@ fn turn_started_emits_turn_started_event() { status: TurnStatus::InProgress, error: None, }, + created_at: 0, })); assert_eq!( @@ -1067,6 +1068,8 @@ fn plan_update_emits_started_then_updated_then_completed() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); assert_eq!( @@ -1123,6 +1126,8 @@ fn plan_update_after_completion_starts_new_todo_list_with_new_id() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1202,6 +1207,8 @@ fn token_usage_update_is_emitted_on_turn_completion() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); assert_eq!( @@ -1237,6 +1244,8 @@ fn turn_completion_recovers_final_message_from_turn_items() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1311,6 +1320,8 @@ fn turn_completion_reconciles_started_items_from_turn_items() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1368,6 +1379,8 @@ fn turn_completion_overwrites_stale_final_message_from_turn_items() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1408,6 +1421,8 @@ fn turn_completion_preserves_streamed_final_message_when_turn_items_are_empty() status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1456,6 +1471,8 @@ fn failed_turn_clears_stale_final_message() { codex_error_info: None, }), }, + completed_at: 0, + duration_ms: None, }, )); @@ -1479,6 +1496,8 @@ fn turn_completion_falls_back_to_final_plan_text() { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); @@ -1527,6 +1546,8 @@ fn turn_failure_prefers_structured_error_message() { status: TurnStatus::Failed, error: None, }, + completed_at: 0, + duration_ms: None, }, )); assert_eq!( diff --git a/codex-rs/tui/src/app.rs b/codex-rs/tui/src/app.rs index e262b55a31..759bf52c7c 100644 --- a/codex-rs/tui/src/app.rs +++ b/codex-rs/tui/src/app.rs @@ -9191,6 +9191,7 @@ guardian_approval = true ServerNotification::TurnStarted(TurnStartedNotification { thread_id: thread_id.to_string(), turn: test_turn(turn_id, TurnStatus::InProgress, Vec::new()), + created_at: 0, }) } @@ -9202,6 +9203,8 @@ guardian_approval = true ServerNotification::TurnCompleted(TurnCompletedNotification { thread_id: thread_id.to_string(), turn: test_turn(turn_id, status, Vec::new()), + completed_at: 0, + duration_ms: None, }) } diff --git a/codex-rs/tui/src/app/app_server_adapter.rs b/codex-rs/tui/src/app/app_server_adapter.rs index ef5a061e32..3a994353b8 100644 --- a/codex-rs/tui/src/app/app_server_adapter.rs +++ b/codex-rs/tui/src/app/app_server_adapter.rs @@ -1104,6 +1104,8 @@ mod tests { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }), ) .expect("notification should bridge"); @@ -1316,6 +1318,8 @@ mod tests { status: TurnStatus::Interrupted, error: None, }, + completed_at: 0, + duration_ms: None, }), ) .expect("notification should bridge"); @@ -1352,6 +1356,8 @@ mod tests { additional_details: None, }), }, + completed_at: 0, + duration_ms: None, }), ) .expect("notification should bridge"); diff --git a/codex-rs/tui/src/app/pending_interactive_replay.rs b/codex-rs/tui/src/app/pending_interactive_replay.rs index 63c8fe1249..408e4d0521 100644 --- a/codex-rs/tui/src/app/pending_interactive_replay.rs +++ b/codex-rs/tui/src/app/pending_interactive_replay.rs @@ -677,6 +677,8 @@ mod tests { status: TurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }) } diff --git a/codex-rs/tui/src/chatwidget.rs b/codex-rs/tui/src/chatwidget.rs index 02429e9d9b..63b201d111 100644 --- a/codex-rs/tui/src/chatwidget.rs +++ b/codex-rs/tui/src/chatwidget.rs @@ -5911,6 +5911,8 @@ impl ChatWidget { status, error, }, + completed_at: 0, + duration_ms: None, }, Some(replay_kind), ); diff --git a/codex-rs/tui/src/chatwidget/tests/app_server.rs b/codex-rs/tui/src/chatwidget/tests/app_server.rs index 2cabed2cc8..a5d1a706fd 100644 --- a/codex-rs/tui/src/chatwidget/tests/app_server.rs +++ b/codex-rs/tui/src/chatwidget/tests/app_server.rs @@ -94,6 +94,7 @@ async fn live_app_server_turn_completed_clears_working_status_after_answer_item( status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); @@ -133,6 +134,8 @@ async fn live_app_server_turn_completed_clears_working_status_after_answer_item( status: AppServerTurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }), /*replay_kind*/ None, ); @@ -416,6 +419,7 @@ async fn live_app_server_failed_turn_does_not_duplicate_error_history() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); @@ -451,6 +455,8 @@ async fn live_app_server_failed_turn_does_not_duplicate_error_history() { additional_details: None, }), }, + completed_at: 0, + duration_ms: None, }), /*replay_kind*/ None, ); @@ -472,6 +478,7 @@ async fn live_app_server_stream_recovery_restores_previous_status_header() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); @@ -526,6 +533,7 @@ async fn live_app_server_server_overloaded_error_renders_warning() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); diff --git a/codex-rs/tui/src/chatwidget/tests/history_replay.rs b/codex-rs/tui/src/chatwidget/tests/history_replay.rs index 58993d67e8..5fc1471f5d 100644 --- a/codex-rs/tui/src/chatwidget/tests/history_replay.rs +++ b/codex-rs/tui/src/chatwidget/tests/history_replay.rs @@ -537,6 +537,7 @@ async fn replayed_retryable_app_server_error_keeps_turn_running() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), Some(ReplayKind::ThreadSnapshot), ); @@ -687,6 +688,7 @@ async fn live_reasoning_summary_is_not_rendered_twice_when_item_completes() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); diff --git a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs index 265339f5d1..7b2c5d8eb6 100644 --- a/codex-rs/tui/src/chatwidget/tests/plan_mode.rs +++ b/codex-rs/tui/src/chatwidget/tests/plan_mode.rs @@ -851,6 +851,7 @@ async fn submit_user_message_queues_while_compaction_turn_is_running() { status: AppServerTurnStatus::InProgress, error: None, }, + created_at: 0, }), /*replay_kind*/ None, ); @@ -894,6 +895,8 @@ async fn submit_user_message_queues_while_compaction_turn_is_running() { status: AppServerTurnStatus::Completed, error: None, }, + completed_at: 0, + duration_ms: None, }), /*replay_kind*/ None, ); From 8068f5dc2ce0dc27ace4a5b2a541313038b2d686 Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Thu, 2 Apr 2026 16:52:32 -0700 Subject: [PATCH 2/3] [codex-analytics] feature plumbing and emittance --- .../analytics/src/analytics_client_tests.rs | 392 ++++++++++++++++++ codex-rs/analytics/src/client.rs | 22 + codex-rs/analytics/src/events.rs | 45 ++ codex-rs/analytics/src/facts.rs | 44 ++ codex-rs/analytics/src/lib.rs | 3 + codex-rs/analytics/src/reducer.rs | 369 ++++++++++++++++- .../app-server/src/bespoke_event_handling.rs | 50 ++- .../app-server/src/codex_message_processor.rs | 10 + codex-rs/app-server/src/message_processor.rs | 22 +- codex-rs/app-server/tests/common/config.rs | 28 ++ codex-rs/app-server/tests/common/lib.rs | 1 + .../app-server/tests/suite/v2/analytics.rs | 35 ++ .../app-server/tests/suite/v2/turn_start.rs | 75 ++++ codex-rs/core/src/codex.rs | 72 +++- codex-rs/core/src/codex_delegate.rs | 1 + codex-rs/core/src/codex_tests.rs | 1 + codex-rs/core/src/codex_tests_guardian.rs | 1 + codex-rs/core/src/state/session.rs | 12 + codex-rs/core/src/thread_manager.rs | 23 + codex-rs/protocol/src/protocol.rs | 8 + 20 files changed, 1176 insertions(+), 38 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index d51112a8bd..d9a080aa1b 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -6,6 +6,7 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::CodexTurnEventRequest; use crate::events::ThreadInitializationMode; use crate::events::ThreadInitializedEvent; use crate::events::ThreadInitializedEventParams; @@ -25,28 +26,43 @@ use crate::facts::PluginUsedInput; use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; use crate::reducer::AnalyticsReducer; use crate::reducer::normalize_path_for_skill_id; use crate::reducer::skill_id_for_local_skill; use codex_app_server_protocol::ApprovalsReviewer as AppServerApprovalsReviewer; use codex_app_server_protocol::AskForApproval as AppServerAskForApproval; use codex_app_server_protocol::ClientInfo; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeCapabilities; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::SandboxPolicy as AppServerSandboxPolicy; +use codex_app_server_protocol::ServerNotification; use codex_app_server_protocol::SessionSource as AppServerSessionSource; use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus; +use codex_app_server_protocol::Turn; +use codex_app_server_protocol::TurnCompletedNotification; +use codex_app_server_protocol::TurnError as AppServerTurnError; +use codex_app_server_protocol::TurnStartParams; +use codex_app_server_protocol::TurnStartedNotification; +use codex_app_server_protocol::TurnStatus as AppServerTurnStatus; +use codex_app_server_protocol::UserInput as AppServerUserInput; use codex_login::default_client::DEFAULT_ORIGINATOR; use codex_login::default_client::originator; use codex_plugin::AppConnectorId; use codex_plugin::PluginCapabilitySummary; use codex_plugin::PluginId; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; use pretty_assertions::assert_eq; use serde_json::json; use std::collections::HashSet; @@ -111,6 +127,173 @@ fn sample_thread_resume_response(thread_id: &str, ephemeral: bool, model: &str) } } +fn sample_turn_start_request(thread_id: &str, request_id: i64) -> ClientRequest { + ClientRequest::TurnStart { + request_id: RequestId::Integer(request_id), + params: TurnStartParams { + thread_id: thread_id.to_string(), + input: vec![ + AppServerUserInput::Text { + text: "hello".to_string(), + text_elements: vec![], + }, + AppServerUserInput::Image { + url: "https://example.com/a.png".to_string(), + }, + ], + ..Default::default() + }, + } +} + +fn sample_turn_start_response(turn_id: &str, request_id: i64) -> ClientResponse { + ClientResponse::TurnStart { + request_id: RequestId::Integer(request_id), + response: codex_app_server_protocol::TurnStartResponse { + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + }, + }, + } +} + +fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNotification { + ServerNotification::TurnStarted(TurnStartedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status: AppServerTurnStatus::InProgress, + error: None, + }, + created_at: 455, + }) +} + +fn sample_turn_completed_notification( + thread_id: &str, + turn_id: &str, + status: AppServerTurnStatus, + codex_error_info: Option, +) -> ServerNotification { + ServerNotification::TurnCompleted(TurnCompletedNotification { + thread_id: thread_id.to_string(), + turn: Turn { + id: turn_id.to_string(), + items: vec![], + status, + error: codex_error_info.map(|codex_error_info| AppServerTurnError { + message: "turn failed".to_string(), + codex_error_info: Some(codex_error_info), + additional_details: None, + }), + }, + completed_at: 456, + duration_ms: Some(1234), + }) +} + +fn sample_turn_resolved_config(turn_id: &str) -> TurnResolvedConfigFact { + TurnResolvedConfigFact { + turn_id: turn_id.to_string(), + thread_id: "thread-2".to_string(), + num_input_images: 1, + submission_type: None, + model: "gpt-5".to_string(), + model_provider: "openai".to_string(), + sandbox_policy: SandboxPolicy::new_read_only_policy(), + reasoning_effort: None, + reasoning_summary: None, + service_tier: None, + approval_policy: AskForApproval::OnRequest, + approvals_reviewer: ApprovalsReviewer::GuardianSubagent, + sandbox_network_access: true, + collaboration_mode: ModeKind::Plan, + personality: None, + is_first_turn: true, + } +} + +async fn ingest_turn_prerequisites( + reducer: &mut AnalyticsReducer, + out: &mut Vec, + include_initialize: bool, + include_resolved_config: bool, + include_started: bool, +) { + if include_initialize { + reducer + .ingest( + AnalyticsFact::Initialize { + connection_id: 7, + params: InitializeParams { + client_info: ClientInfo { + name: "codex-tui".to_string(), + title: None, + version: "1.0.0".to_string(), + }, + capabilities: None, + }, + product_client_id: "codex-tui".to_string(), + runtime: CodexRuntimeMetadata { + codex_rs_version: "0.1.0".to_string(), + runtime_os: "macos".to_string(), + runtime_os_version: "15.3.1".to_string(), + runtime_arch: "aarch64".to_string(), + }, + rpc_transport: AppServerRpcTransport::Stdio, + }, + out, + ) + .await; + } + + reducer + .ingest( + AnalyticsFact::Request { + connection_id: 7, + request_id: RequestId::Integer(3), + request: Box::new(sample_turn_start_request("thread-2", /*request_id*/ 3)), + }, + out, + ) + .await; + reducer + .ingest( + AnalyticsFact::Response { + connection_id: 7, + response: Box::new(sample_turn_start_response("turn-2", /*request_id*/ 3)), + }, + out, + ) + .await; + + if include_resolved_config { + reducer + .ingest( + AnalyticsFact::Custom(CustomAnalyticsFact::TurnResolvedConfig(Box::new( + sample_turn_resolved_config("turn-2"), + ))), + out, + ) + .await; + } + + if include_started { + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_started_notification( + "thread-2", "turn-2", + ))), + out, + ) + .await; + } +} + fn expected_absolute_path(path: &PathBuf) -> String { std::fs::canonicalize(path) .unwrap_or_else(|_| path.to_path_buf()) @@ -671,6 +854,215 @@ async fn reducer_ingests_plugin_state_changed_fact() { ); } +#[test] +fn turn_event_serializes_expected_shape() { + let event = TrackEventRequest::TurnEvent(Box::new(CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: crate::events::CodexTurnEventParams { + thread_id: "thread-2".to_string(), + turn_id: "turn-2".to_string(), + product_client_id: Some("codex-tui".to_string()), + submission_type: None, + model: Some("gpt-5".to_string()), + model_provider: "openai".to_string(), + sandbox_policy: Some("read_only"), + reasoning_effort: Some("high".to_string()), + reasoning_summary: Some("detailed".to_string()), + service_tier: "flex".to_string(), + approval_policy: "on-request".to_string(), + approvals_reviewer: "guardian_subagent".to_string(), + sandbox_network_access: true, + collaboration_mode: Some("plan"), + personality: Some("pragmatic".to_string()), + num_input_images: 2, + is_first_turn: true, + status: Some(TurnStatus::Completed), + turn_error: None, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: Some(1234), + created_at: Some(455), + completed_at: Some(456), + }, + })); + + let payload = serde_json::to_value(&event).expect("serialize turn event"); + + assert_eq!( + payload, + json!({ + "event_type": "codex_turn_event", + "event_params": { + "thread_id": "thread-2", + "turn_id": "turn-2", + "product_client_id": "codex-tui", + "submission_type": null, + "model": "gpt-5", + "model_provider": "openai", + "sandbox_policy": "read_only", + "reasoning_effort": "high", + "reasoning_summary": "detailed", + "service_tier": "flex", + "approval_policy": "on-request", + "approvals_reviewer": "guardian_subagent", + "sandbox_network_access": true, + "collaboration_mode": "plan", + "personality": "pragmatic", + "num_input_images": 2, + "is_first_turn": true, + "status": "completed", + "turn_error": null, + "steer_count": null, + "total_tool_call_count": null, + "shell_command_count": null, + "file_change_count": null, + "mcp_tool_call_count": null, + "dynamic_tool_call_count": null, + "subagent_tool_call_count": null, + "web_search_count": null, + "image_generation_count": null, + "duration_ms": 1234, + "created_at": 455, + "completed_at": 456 + } + }) + ); +} + +#[tokio::test] +async fn turn_lifecycle_emits_turn_event() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ true, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_type"], json!("codex_turn_event")); + assert_eq!(payload["event_params"]["thread_id"], json!("thread-2")); + assert_eq!(payload["event_params"]["turn_id"], json!("turn-2")); + assert_eq!( + payload["event_params"]["product_client_id"], + json!("codex-tui") + ); + assert_eq!(payload["event_params"]["num_input_images"], json!(1)); + assert_eq!(payload["event_params"]["status"], json!("completed")); + assert_eq!(payload["event_params"]["created_at"], json!(455)); + assert_eq!(payload["event_params"]["completed_at"], json!(456)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + +#[tokio::test] +async fn turn_does_not_emit_without_required_prerequisites() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ false, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert_eq!(out.len(), 1); + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!( + payload["event_params"]["product_client_id"], + json!(originator().value) + ); + + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ false, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + assert!(out.is_empty()); +} + +#[tokio::test] +async fn turn_completed_without_started_notification_emits_null_created_at() { + let mut reducer = AnalyticsReducer::default(); + let mut out = Vec::new(); + + ingest_turn_prerequisites( + &mut reducer, + &mut out, + /*include_initialize*/ true, + /*include_resolved_config*/ true, + /*include_started*/ false, + ) + .await; + reducer + .ingest( + AnalyticsFact::Notification(Box::new(sample_turn_completed_notification( + "thread-2", + "turn-2", + AppServerTurnStatus::Completed, + /*codex_error_info*/ None, + ))), + &mut out, + ) + .await; + + let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); + assert_eq!(payload["event_params"]["created_at"], json!(null)); + assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); +} + fn sample_plugin_metadata() -> PluginTelemetryMetadata { PluginTelemetryMetadata { plugin_id: PluginId::parse("sample@test").expect("valid plugin id"), diff --git a/codex-rs/analytics/src/client.rs b/codex-rs/analytics/src/client.rs index b8a543a3b2..a4dc58de36 100644 --- a/codex-rs/analytics/src/client.rs +++ b/codex-rs/analytics/src/client.rs @@ -12,9 +12,13 @@ use crate::facts::PluginStateChangedInput; use crate::facts::SkillInvocation; use crate::facts::SkillInvokedInput; use crate::facts::TrackEventsContext; +use crate::facts::TurnResolvedConfigFact; use crate::reducer::AnalyticsReducer; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; use codex_login::AuthManager; use codex_login::default_client::create_client; use codex_plugin::PluginTelemetryMetadata; @@ -153,6 +157,14 @@ impl AnalyticsEventsClient { ))); } + pub fn track_request(&self, connection_id: u64, request_id: RequestId, request: ClientRequest) { + self.record_fact(AnalyticsFact::Request { + connection_id, + request_id, + request: Box::new(request), + }); + } + pub fn track_app_used(&self, tracking: TrackEventsContext, app: AppInvocation) { if !self.queue.should_enqueue_app_used(&tracking, &app) { return; @@ -171,6 +183,12 @@ impl AnalyticsEventsClient { ))); } + pub fn track_turn_resolved_config(&self, fact: TurnResolvedConfigFact) { + self.record_fact(AnalyticsFact::Custom( + CustomAnalyticsFact::TurnResolvedConfig(Box::new(fact)), + )); + } + pub fn track_plugin_installed(&self, plugin: PluginTelemetryMetadata) { self.record_fact(AnalyticsFact::Custom( CustomAnalyticsFact::PluginStateChanged(PluginStateChangedInput { @@ -220,6 +238,10 @@ impl AnalyticsEventsClient { response: Box::new(response), }); } + + pub fn track_notification(&self, notification: ServerNotification) { + self.record_fact(AnalyticsFact::Notification(Box::new(notification))); + } } async fn send_track_events( diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index 36efb01a69..fcb3d4d09f 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -2,6 +2,9 @@ use crate::facts::AppInvocation; use crate::facts::InvocationType; use crate::facts::PluginState; use crate::facts::TrackEventsContext; +use crate::facts::TurnStatus; +use crate::facts::TurnSubmissionType; +use codex_app_server_protocol::CodexErrorInfo; use codex_login::default_client::originator; use codex_plugin::PluginTelemetryMetadata; use codex_protocol::protocol::SessionSource; @@ -35,6 +38,7 @@ pub(crate) enum TrackEventRequest { ThreadInitialized(ThreadInitializedEvent), AppMentioned(CodexAppMentionedEventRequest), AppUsed(CodexAppUsedEventRequest), + TurnEvent(Box), PluginUsed(CodexPluginUsedEventRequest), PluginInstalled(CodexPluginEventRequest), PluginUninstalled(CodexPluginEventRequest), @@ -120,6 +124,47 @@ pub(crate) struct CodexAppUsedEventRequest { pub(crate) event_params: CodexAppMetadata, } +#[derive(Serialize)] +pub(crate) struct CodexTurnEventParams { + pub(crate) thread_id: String, + pub(crate) turn_id: String, + pub(crate) product_client_id: Option, + pub(crate) submission_type: Option, + pub(crate) model: Option, + pub(crate) model_provider: String, + pub(crate) sandbox_policy: Option<&'static str>, + pub(crate) reasoning_effort: Option, + pub(crate) reasoning_summary: Option, + pub(crate) service_tier: String, + pub(crate) approval_policy: String, + pub(crate) approvals_reviewer: String, + pub(crate) sandbox_network_access: bool, + pub(crate) collaboration_mode: Option<&'static str>, + pub(crate) personality: Option, + pub(crate) num_input_images: usize, + pub(crate) is_first_turn: bool, + pub(crate) status: Option, + pub(crate) turn_error: Option, + pub(crate) steer_count: Option, + pub(crate) total_tool_call_count: Option, + pub(crate) shell_command_count: Option, + pub(crate) file_change_count: Option, + pub(crate) mcp_tool_call_count: Option, + pub(crate) dynamic_tool_call_count: Option, + pub(crate) subagent_tool_call_count: Option, + pub(crate) web_search_count: Option, + pub(crate) image_generation_count: Option, + pub(crate) duration_ms: Option, + pub(crate) created_at: Option, + pub(crate) completed_at: Option, +} + +#[derive(Serialize)] +pub(crate) struct CodexTurnEventRequest { + pub(crate) event_type: &'static str, + pub(crate) event_params: CodexTurnEventParams, +} + #[derive(Serialize)] pub(crate) struct CodexPluginMetadata { pub(crate) plugin_id: Option, diff --git a/codex-rs/analytics/src/facts.rs b/codex-rs/analytics/src/facts.rs index 31b8516e85..1c14135581 100644 --- a/codex-rs/analytics/src/facts.rs +++ b/codex-rs/analytics/src/facts.rs @@ -6,6 +6,14 @@ use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; use codex_plugin::PluginTelemetryMetadata; +use codex_protocol::config_types::ApprovalsReviewer; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::config_types::ServiceTier; +use codex_protocol::openai_models::ReasoningEffort; +use codex_protocol::protocol::AskForApproval; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SkillScope; use serde::Serialize; use std::path::PathBuf; @@ -29,6 +37,41 @@ pub fn build_track_events_context( } } +#[derive(Clone)] +pub struct TurnResolvedConfigFact { + pub turn_id: String, + pub thread_id: String, + pub num_input_images: usize, + pub submission_type: Option, + pub model: String, + pub model_provider: String, + pub sandbox_policy: SandboxPolicy, + pub reasoning_effort: Option, + pub reasoning_summary: Option, + pub service_tier: Option, + pub approval_policy: AskForApproval, + pub approvals_reviewer: ApprovalsReviewer, + pub sandbox_network_access: bool, + pub collaboration_mode: ModeKind, + pub personality: Option, + pub is_first_turn: bool, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnSubmissionType { + Default, + Queued, +} + +#[derive(Clone, Copy, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TurnStatus { + Completed, + Failed, + Interrupted, +} + #[derive(Clone, Debug)] pub struct SkillInvocation { pub skill_name: String, @@ -75,6 +118,7 @@ pub(crate) enum AnalyticsFact { } pub(crate) enum CustomAnalyticsFact { + TurnResolvedConfig(Box), SkillInvoked(SkillInvokedInput), AppMentioned(AppMentionedInput), AppUsed(AppUsedInput), diff --git a/codex-rs/analytics/src/lib.rs b/codex-rs/analytics/src/lib.rs index 6f927d09c0..f74b9dc716 100644 --- a/codex-rs/analytics/src/lib.rs +++ b/codex-rs/analytics/src/lib.rs @@ -9,6 +9,9 @@ pub use facts::AppInvocation; pub use facts::InvocationType; pub use facts::SkillInvocation; pub use facts::TrackEventsContext; +pub use facts::TurnResolvedConfigFact; +pub use facts::TurnStatus; +pub use facts::TurnSubmissionType; pub use facts::build_track_events_context; #[cfg(test)] diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index d83cddcbbc..930ae8bc58 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -5,6 +5,8 @@ use crate::events::CodexAppUsedEventRequest; use crate::events::CodexPluginEventRequest; use crate::events::CodexPluginUsedEventRequest; use crate::events::CodexRuntimeMetadata; +use crate::events::CodexTurnEventParams; +use crate::events::CodexTurnEventRequest; use crate::events::SkillInvocationEventParams; use crate::events::SkillInvocationEventRequest; use crate::events::ThreadInitializationMode; @@ -24,11 +26,22 @@ use crate::facts::PluginState; use crate::facts::PluginStateChangedInput; use crate::facts::PluginUsedInput; use crate::facts::SkillInvokedInput; +use crate::facts::TurnResolvedConfigFact; +use crate::facts::TurnStatus; +use codex_app_server_protocol::ClientRequest; use codex_app_server_protocol::ClientResponse; +use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeParams; +use codex_app_server_protocol::RequestId; +use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::UserInput; use codex_git_utils::collect_git_info; use codex_git_utils::get_git_repo_root; use codex_login::default_client::originator; +use codex_protocol::config_types::ModeKind; +use codex_protocol::config_types::Personality; +use codex_protocol::config_types::ReasoningSummary; +use codex_protocol::protocol::SandboxPolicy; use codex_protocol::protocol::SessionSource; use codex_protocol::protocol::SkillScope; use sha1::Digest; @@ -37,6 +50,8 @@ use std::path::Path; #[derive(Default)] pub(crate) struct AnalyticsReducer { + requests: HashMap<(u64, RequestId), RequestState>, + turns: HashMap, connections: HashMap, } @@ -45,6 +60,32 @@ struct ConnectionState { runtime: CodexRuntimeMetadata, } +enum RequestState { + TurnStart(PendingTurnStartState), +} + +struct PendingTurnStartState { + thread_id: String, + num_input_images: usize, +} + +#[derive(Clone)] +struct CompletedTurnState { + status: Option, + turn_error: Option, + completed_at: u64, + duration_ms: Option, +} + +struct TurnState { + connection_id: Option, + thread_id: Option, + num_input_images: Option, + resolved_config: Option, + created_at: Option, + completed: Option, +} + impl AnalyticsReducer { pub(crate) async fn ingest(&mut self, input: AnalyticsFact, out: &mut Vec) { match input { @@ -64,18 +105,25 @@ impl AnalyticsReducer { ); } AnalyticsFact::Request { - connection_id: _connection_id, - request_id: _request_id, - request: _request, - } => {} + connection_id, + request_id, + request, + } => { + self.ingest_request(connection_id, request_id, *request); + } AnalyticsFact::Response { connection_id, response, } => { self.ingest_response(connection_id, *response, out); } - AnalyticsFact::Notification(_notification) => {} + AnalyticsFact::Notification(notification) => { + self.ingest_notification(*notification, out); + } AnalyticsFact::Custom(input) => match input { + CustomAnalyticsFact::TurnResolvedConfig(input) => { + self.ingest_turn_resolved_config(*input, out); + } CustomAnalyticsFact::SkillInvoked(input) => { self.ingest_skill_invoked(input, out).await; } @@ -120,6 +168,52 @@ impl AnalyticsReducer { ); } + fn ingest_request( + &mut self, + connection_id: u64, + request_id: RequestId, + request: ClientRequest, + ) { + let ClientRequest::TurnStart { params, .. } = request else { + return; + }; + self.requests.insert( + (connection_id, request_id), + RequestState::TurnStart(PendingTurnStartState { + thread_id: params.thread_id, + num_input_images: params + .input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + }), + ); + } + + fn ingest_turn_resolved_config( + &mut self, + input: TurnResolvedConfigFact, + out: &mut Vec, + ) { + let turn_id = input.turn_id.clone(); + let thread_id = input.thread_id.clone(); + let num_input_images = input.num_input_images; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + created_at: None, + completed: None, + }); + turn_state.thread_id = Some(thread_id); + turn_state.num_input_images = Some(num_input_images); + turn_state.resolved_config = Some(input); + self.maybe_emit_turn_event(&turn_id, out); + } + async fn ingest_skill_invoked( &mut self, input: SkillInvokedInput, @@ -220,24 +314,116 @@ impl AnalyticsReducer { response: ClientResponse, out: &mut Vec, ) { - let (thread, model, initialization_mode) = match response { - ClientResponse::ThreadStart { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::New, - ), - ClientResponse::ThreadResume { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Resumed, - ), - ClientResponse::ThreadFork { response, .. } => ( - response.thread, - response.model, - ThreadInitializationMode::Forked, - ), - _ => return, - }; + match response { + ClientResponse::ThreadStart { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::New, + out, + ); + } + ClientResponse::ThreadResume { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Resumed, + out, + ); + } + ClientResponse::ThreadFork { response, .. } => { + self.emit_thread_initialized( + connection_id, + response.thread, + response.model, + ThreadInitializationMode::Forked, + out, + ); + } + ClientResponse::TurnStart { + request_id, + response, + } => { + let turn_id = response.turn.id; + let Some(RequestState::TurnStart(pending_request)) = + self.requests.remove(&(connection_id, request_id)) + else { + return; + }; + let turn_state = self.turns.entry(turn_id.clone()).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + created_at: None, + completed: None, + }); + turn_state.connection_id = Some(connection_id); + turn_state.thread_id = Some(pending_request.thread_id); + turn_state.num_input_images = Some(pending_request.num_input_images); + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn ingest_notification( + &mut self, + notification: ServerNotification, + out: &mut Vec, + ) { + match notification { + ServerNotification::TurnStarted(notification) => { + let turn_state = self.turns.entry(notification.turn.id).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + created_at: None, + completed: None, + }); + turn_state.created_at = u64::try_from(notification.created_at).ok(); + } + ServerNotification::TurnCompleted(notification) => { + let turn_state = + self.turns + .entry(notification.turn.id.clone()) + .or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + created_at: None, + completed: None, + }); + turn_state.completed = Some(CompletedTurnState { + status: analytics_turn_status(notification.turn.status), + turn_error: notification + .turn + .error + .and_then(|error| error.codex_error_info), + completed_at: u64::try_from(notification.completed_at).unwrap_or_default(), + duration_ms: notification + .duration_ms + .and_then(|duration_ms| u64::try_from(duration_ms).ok()), + }); + let turn_id = notification.turn.id; + self.maybe_emit_turn_event(&turn_id, out); + } + _ => {} + } + } + + fn emit_thread_initialized( + &mut self, + connection_id: u64, + thread: codex_app_server_protocol::Thread, + model: String, + initialization_mode: ThreadInitializationMode, + out: &mut Vec, + ) { let thread_source: SessionSource = thread.source.into(); let Some(connection_state) = self.connections.get(&connection_id) else { return; @@ -260,6 +446,143 @@ impl AnalyticsReducer { }, )); } + + fn maybe_emit_turn_event(&mut self, turn_id: &str, out: &mut Vec) { + let Some(turn_state) = self.turns.get(turn_id) else { + return; + }; + if turn_state.thread_id.is_none() + || turn_state.num_input_images.is_none() + || turn_state.resolved_config.is_none() + || turn_state.completed.is_none() + { + return; + } + let product_client_id = turn_state + .connection_id + .and_then(|connection_id| self.connections.get(&connection_id)) + .map(|connection_state| connection_state.app_server_client.product_client_id.clone()) + .unwrap_or_else(|| originator().value); + out.push(TrackEventRequest::TurnEvent(Box::new( + CodexTurnEventRequest { + event_type: "codex_turn_event", + event_params: codex_turn_event_params( + product_client_id, + turn_id.to_string(), + turn_state, + ), + }, + ))); + self.turns.remove(turn_id); + } +} + +fn codex_turn_event_params( + product_client_id: String, + turn_id: String, + turn_state: &TurnState, +) -> CodexTurnEventParams { + let (Some(thread_id), Some(num_input_images), Some(resolved_config), Some(completed)) = ( + turn_state.thread_id.clone(), + turn_state.num_input_images, + turn_state.resolved_config.clone(), + turn_state.completed.clone(), + ) else { + unreachable!("turn event params require a fully populated turn state"); + }; + let created_at = turn_state.created_at; + let TurnResolvedConfigFact { + turn_id: _resolved_turn_id, + thread_id: _resolved_thread_id, + num_input_images: _resolved_num_input_images, + submission_type, + model, + model_provider, + sandbox_policy, + reasoning_effort, + reasoning_summary, + service_tier, + approval_policy, + approvals_reviewer, + sandbox_network_access, + collaboration_mode, + personality, + is_first_turn, + } = resolved_config; + CodexTurnEventParams { + thread_id, + turn_id, + product_client_id: Some(product_client_id), + submission_type, + model: Some(model), + model_provider, + sandbox_policy: Some(sandbox_policy_mode(&sandbox_policy)), + reasoning_effort: reasoning_effort.map(|value| value.to_string()), + reasoning_summary: reasoning_summary_mode(reasoning_summary), + service_tier: service_tier + .map(|value| value.to_string()) + .unwrap_or_else(|| "default".to_string()), + approval_policy: approval_policy.to_string(), + approvals_reviewer: approvals_reviewer.to_string(), + sandbox_network_access, + collaboration_mode: Some(collaboration_mode_mode(collaboration_mode)), + personality: personality_mode(personality), + num_input_images, + is_first_turn, + status: completed.status, + turn_error: completed.turn_error, + steer_count: None, + total_tool_call_count: None, + shell_command_count: None, + file_change_count: None, + mcp_tool_call_count: None, + dynamic_tool_call_count: None, + subagent_tool_call_count: None, + web_search_count: None, + image_generation_count: None, + duration_ms: completed.duration_ms, + created_at, + completed_at: Some(completed.completed_at), + } +} + +fn sandbox_policy_mode(sandbox_policy: &SandboxPolicy) -> &'static str { + match sandbox_policy { + SandboxPolicy::DangerFullAccess => "full_access", + SandboxPolicy::ReadOnly { .. } => "read_only", + SandboxPolicy::WorkspaceWrite { .. } => "workspace_write", + SandboxPolicy::ExternalSandbox { .. } => "external_sandbox", + } +} + +fn collaboration_mode_mode(mode: ModeKind) -> &'static str { + match mode { + ModeKind::Plan => "plan", + ModeKind::Default | ModeKind::PairProgramming | ModeKind::Execute => "default", + } +} + +fn reasoning_summary_mode(summary: Option) -> Option { + match summary { + Some(ReasoningSummary::None) | None => None, + Some(summary) => Some(summary.to_string()), + } +} + +fn personality_mode(personality: Option) -> Option { + match personality { + Some(Personality::None) | None => None, + Some(personality) => Some(personality.to_string()), + } +} + +fn analytics_turn_status(status: codex_app_server_protocol::TurnStatus) -> Option { + match status { + codex_app_server_protocol::TurnStatus::Completed => Some(TurnStatus::Completed), + codex_app_server_protocol::TurnStatus::Failed => Some(TurnStatus::Failed), + codex_app_server_protocol::TurnStatus::Interrupted => Some(TurnStatus::Interrupted), + codex_app_server_protocol::TurnStatus::InProgress => None, + } } pub(crate) fn skill_id_for_local_skill( diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 6e288da81e..04d3465f08 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -12,6 +12,7 @@ use crate::thread_state::ThreadState; use crate::thread_state::TurnSummary; use crate::thread_status::ThreadWatchActiveGuard; use crate::thread_status::ThreadWatchManager; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::AccountRateLimitsUpdatedNotification; use codex_app_server_protocol::AdditionalPermissionProfile as V2AdditionalPermissionProfile; use codex_app_server_protocol::AgentMessageDeltaNotification; @@ -260,6 +261,7 @@ pub(crate) async fn apply_bespoke_event_handling( conversation_id: ThreadId, conversation: Arc, thread_manager: Arc, + analytics_events_client: AnalyticsEventsClient, outgoing: ThreadScopedOutgoingMessageSender, thread_state: Arc>, thread_watch_manager: ThreadWatchManager, @@ -293,6 +295,8 @@ pub(crate) async fn apply_bespoke_event_handling( turn, created_at: set_turn_started_at(&thread_state).await, }; + analytics_events_client + .track_notification(ServerNotification::TurnStarted(notification.clone())); outgoing .send_server_notification(ServerNotification::TurnStarted(notification)) .await; @@ -305,7 +309,14 @@ pub(crate) async fn apply_bespoke_event_handling( thread_watch_manager .note_turn_completed(&conversation_id.to_string(), turn_failed) .await; - handle_turn_complete(conversation_id, event_turn_id, &outgoing, &thread_state).await; + handle_turn_complete( + conversation_id, + event_turn_id, + Some(&analytics_events_client), + &outgoing, + &thread_state, + ) + .await; } EventMsg::SkillsUpdateAvailable => { if let ApiVersion::V2 = api_version { @@ -1951,6 +1962,7 @@ async fn emit_turn_completed_with_status( status: TurnStatus, error: Option, started_at_ms: Option, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, ) { let completed_at_ms = now_unix_timestamp_millis(); @@ -1966,6 +1978,10 @@ async fn emit_turn_completed_with_status( duration_ms: started_at_ms .map(|started_at_ms| completed_at_ms.saturating_sub(started_at_ms)), }; + if let Some(analytics_events_client) = analytics_events_client { + analytics_events_client + .track_notification(ServerNotification::TurnCompleted(notification.clone())); + } outgoing .send_server_notification(ServerNotification::TurnCompleted(notification)) .await; @@ -2109,6 +2125,7 @@ async fn find_and_remove_turn_summary( async fn handle_turn_complete( conversation_id: ThreadId, event_turn_id: String, + analytics_events_client: Option<&AnalyticsEventsClient>, outgoing: &ThreadScopedOutgoingMessageSender, thread_state: &Arc>, ) { @@ -2126,6 +2143,7 @@ async fn handle_turn_complete( status, error, started_at_ms, + analytics_events_client, outgoing, ) .await; @@ -2145,6 +2163,7 @@ async fn handle_turn_interrupted( TurnStatus::Interrupted, /*error*/ None, turn_summary.started_at_ms, + /*analytics_events_client*/ None, outgoing, ) .await; @@ -3404,6 +3423,8 @@ mod tests { handle_turn_complete( conversation_id, event_turn_id.clone(), + /*analytics_events_client*/ None, + /*analytics_events_client*/ None, &outgoing, &thread_state, ) @@ -3664,7 +3685,14 @@ mod tests { &thread_state, ) .await; - handle_turn_complete(conversation_a, a_turn1.clone(), &outgoing, &thread_state).await; + handle_turn_complete( + conversation_a, + a_turn1.clone(), + /*analytics_events_client*/ None, + &outgoing, + &thread_state, + ) + .await; // Turn 1 on conversation B let b_turn1 = "b_turn1".to_string(); @@ -3678,11 +3706,25 @@ mod tests { &thread_state, ) .await; - handle_turn_complete(conversation_b, b_turn1.clone(), &outgoing, &thread_state).await; + handle_turn_complete( + conversation_b, + b_turn1.clone(), + /*analytics_events_client*/ None, + &outgoing, + &thread_state, + ) + .await; // Turn 2 on conversation A let a_turn2 = "a_turn2".to_string(); - handle_turn_complete(conversation_a, a_turn2.clone(), &outgoing, &thread_state).await; + handle_turn_complete( + conversation_a, + a_turn2.clone(), + /*analytics_events_client*/ None, + &outgoing, + &thread_state, + ) + .await; // Verify: A turn 1 let msg = recv_broadcast_message(&mut rx).await?; diff --git a/codex-rs/app-server/src/codex_message_processor.rs b/codex-rs/app-server/src/codex_message_processor.rs index e5d51dd474..18a8c0b6b0 100644 --- a/codex-rs/app-server/src/codex_message_processor.rs +++ b/codex-rs/app-server/src/codex_message_processor.rs @@ -6449,6 +6449,15 @@ impl CodexMessageProcessor { }; let response = TurnStartResponse { turn }; + if self.config.features.enabled(Feature::GeneralAnalytics) { + self.analytics_events_client.track_response( + request_id.connection_id.0, + ClientResponse::TurnStart { + request_id: request_id.request_id.clone(), + response: response.clone(), + }, + ); + } self.outgoing.send_response(request_id, response).await; } Err(err) => { @@ -7221,6 +7230,7 @@ impl CodexMessageProcessor { conversation_id, conversation.clone(), thread_manager.clone(), + listener_task_context.analytics_events_client.clone(), thread_outgoing, thread_state.clone(), thread_watch_manager.clone(), diff --git a/codex-rs/app-server/src/message_processor.rs b/codex-rs/app-server/src/message_processor.rs index c63d0a83cf..37dd59a18f 100644 --- a/codex-rs/app-server/src/message_processor.rs +++ b/codex-rs/app-server/src/message_processor.rs @@ -224,7 +224,12 @@ impl MessageProcessor { outgoing: outgoing.clone(), }), ); - let thread_manager = Arc::new(ThreadManager::new( + let analytics_events_client = AnalyticsEventsClient::new( + Arc::clone(&auth_manager), + config.chatgpt_base_url.trim_end_matches('/').to_string(), + config.analytics_enabled, + ); + let thread_manager = Arc::new(ThreadManager::new_with_analytics_events_client( config.as_ref(), auth_manager.clone(), session_source, @@ -234,13 +239,9 @@ impl MessageProcessor { .enabled(Feature::DefaultModeRequestUserInput), }, environment_manager, + Some(analytics_events_client.clone()), )); auth_manager.set_forced_chatgpt_workspace_id(config.forced_chatgpt_workspace_id.clone()); - let analytics_events_client = AnalyticsEventsClient::new( - Arc::clone(&auth_manager), - config.chatgpt_base_url.trim_end_matches('/').to_string(), - config.analytics_enabled, - ); thread_manager .plugins_manager() .set_analytics_events_client(analytics_events_client.clone()); @@ -677,6 +678,15 @@ impl MessageProcessor { self.outgoing.send_error(connection_request_id, error).await; return; } + if self.config.features.enabled(Feature::GeneralAnalytics) + && let ClientRequest::TurnStart { request_id, .. } = &codex_request + { + self.analytics_events_client.track_request( + connection_id.0, + request_id.clone(), + codex_request.clone(), + ); + } match codex_request { ClientRequest::ConfigRead { request_id, params } => { diff --git a/codex-rs/app-server/tests/common/config.rs b/codex-rs/app-server/tests/common/config.rs index deb16c6322..1ac2572fa2 100644 --- a/codex-rs/app-server/tests/common/config.rs +++ b/codex-rs/app-server/tests/common/config.rs @@ -78,3 +78,31 @@ model_provider = "{model_provider_id}" ), ) } + +pub fn write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home: &Path, + server_uri: &str, + chatgpt_base_url: &str, +) -> std::io::Result<()> { + let config_toml = codex_home.join("config.toml"); + std::fs::write( + config_toml, + format!( + r#" +model = "mock-model" +approval_policy = "never" +sandbox_mode = "read-only" +chatgpt_base_url = "{chatgpt_base_url}" + +model_provider = "mock_provider" + +[model_providers.mock_provider] +name = "Mock provider for test" +base_url = "{server_uri}/v1" +wire_api = "responses" +request_max_retries = 0 +stream_max_retries = 0 +"# + ), + ) +} diff --git a/codex-rs/app-server/tests/common/lib.rs b/codex-rs/app-server/tests/common/lib.rs index 3f89765851..90553760d9 100644 --- a/codex-rs/app-server/tests/common/lib.rs +++ b/codex-rs/app-server/tests/common/lib.rs @@ -14,6 +14,7 @@ pub use auth_fixtures::encode_id_token; pub use auth_fixtures::write_chatgpt_auth; use codex_app_server_protocol::JSONRPCResponse; pub use config::write_mock_responses_config_toml; +pub use config::write_mock_responses_config_toml_with_chatgpt_base_url; pub use core_test_support::format_with_current_shell; pub use core_test_support::format_with_current_shell_display; pub use core_test_support::format_with_current_shell_display_non_login; diff --git a/codex-rs/app-server/tests/suite/v2/analytics.rs b/codex-rs/app-server/tests/suite/v2/analytics.rs index 8e8e328a84..bb2bdb8f0c 100644 --- a/codex-rs/app-server/tests/suite/v2/analytics.rs +++ b/codex-rs/app-server/tests/suite/v2/analytics.rs @@ -120,6 +120,41 @@ pub(crate) async fn wait_for_analytics_payload( serde_json::from_slice(&body).map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}")) } +pub(crate) async fn wait_for_analytics_event( + server: &MockServer, + read_timeout: Duration, + event_type: &str, +) -> Result { + timeout(read_timeout, async { + loop { + let Some(requests) = server.received_requests().await else { + tokio::time::sleep(Duration::from_millis(25)).await; + continue; + }; + for request in &requests { + if request.method != "POST" + || request.url.path() != "/codex/analytics-events/events" + { + continue; + } + let payload: Value = serde_json::from_slice(&request.body) + .map_err(|err| anyhow::anyhow!("invalid analytics payload: {err}"))?; + let Some(events) = payload["events"].as_array() else { + continue; + }; + if let Some(event) = events + .iter() + .find(|event| event["event_type"] == event_type) + { + return Ok::(event.clone()); + } + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await? +} + pub(crate) fn thread_initialized_event(payload: &Value) -> Result<&Value> { let events = payload["events"] .as_array() diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 3155a4be9c..5b612791d2 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use app_test_support::DEFAULT_CLIENT_NAME; use app_test_support::McpProcess; use app_test_support::create_apply_patch_sse_response; use app_test_support::create_exec_command_sse_response; @@ -9,6 +10,7 @@ use app_test_support::create_mock_responses_server_sequence_unchecked; use app_test_support::create_shell_command_sse_response; use app_test_support::format_with_current_shell_display; use app_test_support::to_response; +use app_test_support::write_mock_responses_config_toml_with_chatgpt_base_url; use codex_app_server::INPUT_TOO_LARGE_ERROR_CODE; use codex_app_server::INVALID_PARAMS_ERROR_CODE; use codex_app_server_protocol::ByteRange; @@ -64,6 +66,9 @@ use std::path::Path; use tempfile::TempDir; use tokio::time::timeout; +use super::analytics::enable_analytics_capture; +use super::analytics::wait_for_analytics_event; + #[cfg(windows)] const DEFAULT_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(25); #[cfg(not(windows))] @@ -238,6 +243,76 @@ async fn turn_start_emits_user_message_item_with_text_elements() -> Result<()> { Ok(()) } +#[tokio::test] +async fn turn_start_tracks_turn_event_analytics() -> Result<()> { + let responses = vec![create_final_assistant_message_sse_response("Done")?]; + let server = create_mock_responses_server_sequence_unchecked(responses).await; + + let codex_home = TempDir::new()?; + write_mock_responses_config_toml_with_chatgpt_base_url( + codex_home.path(), + &server.uri(), + &server.uri(), + )?; + enable_analytics_capture(&server, codex_home.path()).await?; + + let mut mcp = McpProcess::new(codex_home.path()).await?; + timeout(DEFAULT_READ_TIMEOUT, mcp.initialize()).await??; + + let thread_req = mcp + .send_thread_start_request(ThreadStartParams { + model: Some("mock-model".to_string()), + ..Default::default() + }) + .await?; + let thread_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(thread_req)), + ) + .await??; + let ThreadStartResponse { thread, .. } = to_response::(thread_resp)?; + + let turn_req = mcp + .send_turn_start_request(TurnStartParams { + thread_id: thread.id.clone(), + input: vec![V2UserInput::Image { + url: "https://example.com/a.png".to_string(), + }], + ..Default::default() + }) + .await?; + let turn_resp: JSONRPCResponse = timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_response_message(RequestId::Integer(turn_req)), + ) + .await??; + let TurnStartResponse { turn } = to_response::(turn_resp)?; + + timeout( + DEFAULT_READ_TIMEOUT, + mcp.read_stream_until_notification_message("turn/completed"), + ) + .await??; + + let event = wait_for_analytics_event(&server, DEFAULT_READ_TIMEOUT, "codex_turn_event").await?; + assert_eq!(event["event_params"]["thread_id"], thread.id); + assert_eq!(event["event_params"]["turn_id"], turn.id); + assert_eq!( + event["event_params"]["product_client_id"], + DEFAULT_CLIENT_NAME + ); + assert_eq!(event["event_params"]["model"], "mock-model"); + assert_eq!(event["event_params"]["model_provider"], "mock_provider"); + assert_eq!(event["event_params"]["sandbox_policy"], "read_only"); + assert_eq!(event["event_params"]["num_input_images"], 1); + assert_eq!(event["event_params"]["status"], "completed"); + assert!(event["event_params"]["created_at"].as_u64().is_some()); + assert!(event["event_params"]["completed_at"].as_u64().is_some()); + assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + + Ok(()) +} + #[tokio::test] async fn turn_start_accepts_text_at_limit_with_mention_item() -> Result<()> { let responses = vec![create_final_assistant_message_sse_response("Done")?]; diff --git a/codex-rs/core/src/codex.rs b/codex-rs/core/src/codex.rs index 73f5c6ed7d..1fa969ed84 100644 --- a/codex-rs/core/src/codex.rs +++ b/codex-rs/core/src/codex.rs @@ -53,6 +53,7 @@ use chrono::Utc; use codex_analytics::AnalyticsEventsClient; use codex_analytics::AppInvocation; use codex_analytics::InvocationType; +use codex_analytics::TurnResolvedConfigFact; use codex_analytics::build_track_events_context; use codex_app_server_protocol::McpServerElicitationRequest; use codex_app_server_protocol::McpServerElicitationRequestParams; @@ -183,6 +184,7 @@ use crate::config::StartedNetworkProxy; use crate::config::resolve_web_search_mode_for_turn; use crate::context_manager::ContextManager; use crate::context_manager::TotalTokenUsageBreakdown; +use crate::context_manager::is_user_turn_boundary; use crate::environment_context::EnvironmentContext; use crate::error::CodexErr; use crate::error::Result as CodexResult; @@ -426,6 +428,7 @@ pub(crate) struct CodexSpawnArgs { pub(crate) inherited_exec_policy: Option>, pub(crate) user_shell_override: Option, pub(crate) parent_trace: Option, + pub(crate) analytics_events_client: Option, } pub(crate) const INITIAL_SUBMIT_ID: &str = ""; @@ -480,6 +483,7 @@ impl Codex { user_shell_override, inherited_exec_policy, parent_trace: _, + analytics_events_client, } = args; let (tx_sub, rx_sub) = async_channel::bounded(SUBMISSION_CHANNEL_CAPACITY); let (tx_event, rx_event) = async_channel::unbounded(); @@ -663,6 +667,7 @@ impl Codex { mcp_manager.clone(), skills_watcher, agent_control, + analytics_events_client, ) .await .map_err(|e| { @@ -800,6 +805,17 @@ pub(crate) fn session_loop_termination_from_handle( .shared() } +fn initial_history_has_prior_user_turns(conversation_history: &InitialHistory) -> bool { + conversation_history.scan_rollout_items(rollout_item_is_user_turn_boundary) +} + +fn rollout_item_is_user_turn_boundary(item: &RolloutItem) -> bool { + match item { + RolloutItem::ResponseItem(item) => is_user_turn_boundary(item), + _ => false, + } +} + /// Context for an initialized model agent /// /// A session has at most 1 running task at a time, and can be interrupted by user input. @@ -1485,6 +1501,7 @@ impl Session { mcp_manager: Arc, skills_watcher: Arc, agent_control: AgentControl, + analytics_events_client: Option, ) -> anyhow::Result> { debug!( "Configuring session: model={}; provider={:?}", @@ -1877,11 +1894,13 @@ impl Session { ), shell_zsh_path: config.zsh_path.clone(), main_execve_wrapper_exe: config.main_execve_wrapper_exe.clone(), - analytics_events_client: AnalyticsEventsClient::new( - Arc::clone(&auth_manager), - config.chatgpt_base_url.trim_end_matches('/').to_string(), - config.analytics_enabled, - ), + analytics_events_client: analytics_events_client.unwrap_or_else(|| { + AnalyticsEventsClient::new( + Arc::clone(&auth_manager), + config.chatgpt_base_url.trim_end_matches('/').to_string(), + config.analytics_enabled, + ) + }), hooks, rollout: Mutex::new(rollout_recorder), user_shell: Arc::new(default_shell), @@ -2192,6 +2211,11 @@ impl Session { SessionSource::SubAgent(_) ) }; + let has_prior_user_turns = initial_history_has_prior_user_turns(&conversation_history); + { + let mut state = self.state.lock().await; + state.set_next_turn_is_first(!has_prior_user_turns); + } match conversation_history { InitialHistory::New => { // Defer initial context insertion until the first real turn starts so @@ -5918,6 +5942,8 @@ pub(crate) async fn run_turn( .await; } + track_turn_resolved_config_analytics(&sess, &turn_context, &input).await; + let skills_outcome = Some(turn_context.turn_skills.outcome.as_ref()); sess.maybe_start_ghost_snapshot(Arc::clone(&turn_context), cancellation_token.child_token()) .await; @@ -6203,6 +6229,42 @@ pub(crate) async fn run_turn( last_agent_message } +async fn track_turn_resolved_config_analytics( + sess: &Session, + turn_context: &TurnContext, + input: &[UserInput], +) { + let is_first_turn = { + let mut state = sess.state.lock().await; + state.take_next_turn_is_first() + }; + sess.services + .analytics_events_client + .track_turn_resolved_config(TurnResolvedConfigFact { + turn_id: turn_context.sub_id.clone(), + thread_id: sess.conversation_id.to_string(), + num_input_images: input + .iter() + .filter(|item| { + matches!(item, UserInput::Image { .. } | UserInput::LocalImage { .. }) + }) + .count(), + submission_type: None, + model: turn_context.model_info.slug.clone(), + model_provider: turn_context.config.model_provider_id.clone(), + sandbox_policy: turn_context.sandbox_policy.get().clone(), + reasoning_effort: turn_context.reasoning_effort, + reasoning_summary: Some(turn_context.reasoning_summary), + service_tier: turn_context.config.service_tier, + approval_policy: turn_context.approval_policy.value(), + approvals_reviewer: turn_context.config.approvals_reviewer, + sandbox_network_access: turn_context.network_sandbox_policy.is_enabled(), + collaboration_mode: turn_context.collaboration_mode.mode, + personality: turn_context.personality, + is_first_turn, + }); +} + async fn run_pre_sampling_compact( sess: &Arc, turn_context: &Arc, diff --git a/codex-rs/core/src/codex_delegate.rs b/codex-rs/core/src/codex_delegate.rs index 6bb8257f09..1fadfdca63 100644 --- a/codex-rs/core/src/codex_delegate.rs +++ b/codex-rs/core/src/codex_delegate.rs @@ -94,6 +94,7 @@ pub(crate) async fn run_codex_thread_interactive( user_shell_override: None, inherited_exec_policy: Some(Arc::clone(&parent_session.services.exec_policy)), parent_trace: None, + analytics_events_client: Some(parent_session.services.analytics_events_client.clone()), }) .await?; let codex = Arc::new(codex); diff --git a/codex-rs/core/src/codex_tests.rs b/codex-rs/core/src/codex_tests.rs index 4ff31e8bd0..986e711348 100644 --- a/codex-rs/core/src/codex_tests.rs +++ b/codex-rs/core/src/codex_tests.rs @@ -2559,6 +2559,7 @@ async fn session_new_fails_when_zsh_fork_enabled_without_zsh_path() { mcp_manager, Arc::new(SkillsWatcher::noop()), AgentControl::default(), + /*analytics_events_client*/ None, ) .await; diff --git a/codex-rs/core/src/codex_tests_guardian.rs b/codex-rs/core/src/codex_tests_guardian.rs index 4f60c2f28e..7738cbbe91 100644 --- a/codex-rs/core/src/codex_tests_guardian.rs +++ b/codex-rs/core/src/codex_tests_guardian.rs @@ -457,6 +457,7 @@ async fn guardian_subagent_does_not_inherit_parent_exec_policy_rules() { inherited_exec_policy: Some(Arc::new(parent_exec_policy)), user_shell_override: None, parent_trace: None, + analytics_events_client: None, }) .await .expect("spawn guardian subagent"); diff --git a/codex-rs/core/src/state/session.rs b/codex-rs/core/src/state/session.rs index 206f75060c..4360b16de4 100644 --- a/codex-rs/core/src/state/session.rs +++ b/codex-rs/core/src/state/session.rs @@ -33,6 +33,7 @@ pub(crate) struct SessionState { pub(crate) active_connector_selection: HashSet, pub(crate) pending_session_start_source: Option, granted_permissions: Option, + next_turn_is_first: bool, } impl SessionState { @@ -51,6 +52,7 @@ impl SessionState { active_connector_selection: HashSet::new(), pending_session_start_source: None, granted_permissions: None, + next_turn_is_first: true, } } @@ -73,6 +75,16 @@ impl SessionState { self.previous_turn_settings = previous_turn_settings; } + pub(crate) fn set_next_turn_is_first(&mut self, value: bool) { + self.next_turn_is_first = value; + } + + pub(crate) fn take_next_turn_is_first(&mut self) -> bool { + let is_first_turn = self.next_turn_is_first; + self.next_turn_is_first = false; + is_first_turn + } + pub(crate) fn clone_history(&self) -> ContextManager { self.history.clone() } diff --git a/codex-rs/core/src/thread_manager.rs b/codex-rs/core/src/thread_manager.rs index 94a81d6209..a5088c4113 100644 --- a/codex-rs/core/src/thread_manager.rs +++ b/codex-rs/core/src/thread_manager.rs @@ -21,6 +21,7 @@ use crate::shell_snapshot::ShellSnapshot; use crate::skills_watcher::SkillsWatcher; use crate::skills_watcher::SkillsWatcherEvent; use crate::tasks::interrupted_turn_history_marker; +use codex_analytics::AnalyticsEventsClient; use codex_app_server_protocol::ThreadHistoryBuilder; use codex_app_server_protocol::TurnStatus; use codex_exec_server::EnvironmentManager; @@ -207,6 +208,7 @@ pub(crate) struct ThreadManagerState { mcp_manager: Arc, skills_watcher: Arc, session_source: SessionSource, + analytics_events_client: Option, // Captures submitted ops for testing purpose when test mode is enabled. ops_log: Option, } @@ -218,6 +220,24 @@ impl ThreadManager { session_source: SessionSource, collaboration_modes_config: CollaborationModesConfig, environment_manager: Arc, + ) -> Self { + Self::new_with_analytics_events_client( + config, + auth_manager, + session_source, + collaboration_modes_config, + environment_manager, + /*analytics_events_client*/ None, + ) + } + + pub fn new_with_analytics_events_client( + config: &Config, + auth_manager: Arc, + session_source: SessionSource, + collaboration_modes_config: CollaborationModesConfig, + environment_manager: Arc, + analytics_events_client: Option, ) -> Self { let codex_home = config.codex_home.clone(); let restriction_product = session_source.restriction_product(); @@ -256,6 +276,7 @@ impl ThreadManager { skills_watcher, auth_manager, session_source, + analytics_events_client, ops_log: should_use_test_thread_manager_behavior() .then(|| Arc::new(std::sync::Mutex::new(Vec::new()))), }), @@ -325,6 +346,7 @@ impl ThreadManager { skills_watcher, auth_manager, session_source: SessionSource::Exec, + analytics_events_client: None, ops_log: should_use_test_thread_manager_behavior() .then(|| Arc::new(std::sync::Mutex::new(Vec::new()))), }), @@ -869,6 +891,7 @@ impl ThreadManagerState { inherited_exec_policy, user_shell_override, parent_trace, + analytics_events_client: self.analytics_events_client.clone(), }) .await?; self.finalize_thread_spawn(codex, thread_id, watch_registration) diff --git a/codex-rs/protocol/src/protocol.rs b/codex-rs/protocol/src/protocol.rs index 5e0c6010e4..e0903fd5eb 100644 --- a/codex-rs/protocol/src/protocol.rs +++ b/codex-rs/protocol/src/protocol.rs @@ -2263,6 +2263,14 @@ pub enum InitialHistory { } impl InitialHistory { + pub fn scan_rollout_items(&self, mut predicate: impl FnMut(&RolloutItem) -> bool) -> bool { + match self { + InitialHistory::New => false, + InitialHistory::Resumed(resumed) => resumed.history.iter().any(&mut predicate), + InitialHistory::Forked(items) => items.iter().any(predicate), + } + } + pub fn forked_from_id(&self) -> Option { match self { InitialHistory::New => None, From d180b0846ce44785993b4ef7bf9f18b4d344ecde Mon Sep 17 00:00:00 2001 From: rhan-oai Date: Thu, 2 Apr 2026 17:11:09 -0700 Subject: [PATCH 3/3] [codex-analytics] add token usage metadata --- .../analytics/src/analytics_client_tests.rs | 72 +++++++++++++++++++ codex-rs/analytics/src/events.rs | 5 ++ codex-rs/analytics/src/reducer.rs | 34 +++++++++ .../app-server/src/bespoke_event_handling.rs | 18 ++++- .../app-server/tests/suite/v2/turn_start.rs | 5 ++ 5 files changed, 132 insertions(+), 2 deletions(-) diff --git a/codex-rs/analytics/src/analytics_client_tests.rs b/codex-rs/analytics/src/analytics_client_tests.rs index d9a080aa1b..3a0230113c 100644 --- a/codex-rs/analytics/src/analytics_client_tests.rs +++ b/codex-rs/analytics/src/analytics_client_tests.rs @@ -46,6 +46,9 @@ use codex_app_server_protocol::Thread; use codex_app_server_protocol::ThreadResumeResponse; use codex_app_server_protocol::ThreadStartResponse; use codex_app_server_protocol::ThreadStatus as AppServerThreadStatus; +use codex_app_server_protocol::ThreadTokenUsage; +use codex_app_server_protocol::ThreadTokenUsageUpdatedNotification; +use codex_app_server_protocol::TokenUsageBreakdown; use codex_app_server_protocol::Turn; use codex_app_server_protocol::TurnCompletedNotification; use codex_app_server_protocol::TurnError as AppServerTurnError; @@ -173,6 +176,33 @@ fn sample_turn_started_notification(thread_id: &str, turn_id: &str) -> ServerNot }) } +fn sample_thread_token_usage_updated_notification( + thread_id: &str, + turn_id: &str, +) -> ServerNotification { + ServerNotification::ThreadTokenUsageUpdated(ThreadTokenUsageUpdatedNotification { + thread_id: thread_id.to_string(), + turn_id: turn_id.to_string(), + token_usage: ThreadTokenUsage { + total: TokenUsageBreakdown { + total_tokens: 500, + input_tokens: 200, + cached_input_tokens: 50, + output_tokens: 220, + reasoning_output_tokens: 30, + }, + last: TokenUsageBreakdown { + total_tokens: 321, + input_tokens: 123, + cached_input_tokens: 45, + output_tokens: 140, + reasoning_output_tokens: 13, + }, + model_context_window: Some(200_000), + }, + }) +} + fn sample_turn_completed_notification( thread_id: &str, turn_id: &str, @@ -223,6 +253,7 @@ async fn ingest_turn_prerequisites( include_initialize: bool, include_resolved_config: bool, include_started: bool, + include_token_usage: bool, ) { if include_initialize { reducer @@ -292,6 +323,17 @@ async fn ingest_turn_prerequisites( ) .await; } + + if include_token_usage { + reducer + .ingest( + AnalyticsFact::Notification(Box::new( + sample_thread_token_usage_updated_notification("thread-2", "turn-2"), + )), + out, + ) + .await; + } } fn expected_absolute_path(path: &PathBuf) -> String { @@ -887,6 +929,11 @@ fn turn_event_serializes_expected_shape() { subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: None, + cached_input_tokens: None, + output_tokens: None, + reasoning_output_tokens: None, + total_tokens: None, duration_ms: Some(1234), created_at: Some(455), completed_at: Some(456), @@ -928,6 +975,11 @@ fn turn_event_serializes_expected_shape() { "subagent_tool_call_count": null, "web_search_count": null, "image_generation_count": null, + "input_tokens": null, + "cached_input_tokens": null, + "output_tokens": null, + "reasoning_output_tokens": null, + "total_tokens": null, "duration_ms": 1234, "created_at": 455, "completed_at": 456 @@ -947,6 +999,7 @@ async fn turn_lifecycle_emits_turn_event() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ true, + /*include_token_usage*/ true, ) .await; reducer @@ -975,6 +1028,14 @@ async fn turn_lifecycle_emits_turn_event() { assert_eq!(payload["event_params"]["created_at"], json!(455)); assert_eq!(payload["event_params"]["completed_at"], json!(456)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(123)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(45)); + assert_eq!(payload["event_params"]["output_tokens"], json!(140)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(13) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(321)); } #[tokio::test] @@ -988,6 +1049,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ false, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1017,6 +1079,7 @@ async fn turn_does_not_emit_without_required_prerequisites() { /*include_initialize*/ true, /*include_resolved_config*/ false, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1044,6 +1107,7 @@ async fn turn_completed_without_started_notification_emits_null_created_at() { /*include_initialize*/ true, /*include_resolved_config*/ true, /*include_started*/ false, + /*include_token_usage*/ false, ) .await; reducer @@ -1061,6 +1125,14 @@ async fn turn_completed_without_started_notification_emits_null_created_at() { let payload = serde_json::to_value(&out[0]).expect("serialize turn event"); assert_eq!(payload["event_params"]["created_at"], json!(null)); assert_eq!(payload["event_params"]["duration_ms"], json!(1234)); + assert_eq!(payload["event_params"]["input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["cached_input_tokens"], json!(null)); + assert_eq!(payload["event_params"]["output_tokens"], json!(null)); + assert_eq!( + payload["event_params"]["reasoning_output_tokens"], + json!(null) + ); + assert_eq!(payload["event_params"]["total_tokens"], json!(null)); } fn sample_plugin_metadata() -> PluginTelemetryMetadata { diff --git a/codex-rs/analytics/src/events.rs b/codex-rs/analytics/src/events.rs index fcb3d4d09f..3ae13b0a53 100644 --- a/codex-rs/analytics/src/events.rs +++ b/codex-rs/analytics/src/events.rs @@ -154,6 +154,11 @@ pub(crate) struct CodexTurnEventParams { pub(crate) subagent_tool_call_count: Option, pub(crate) web_search_count: Option, pub(crate) image_generation_count: Option, + pub(crate) input_tokens: Option, + pub(crate) cached_input_tokens: Option, + pub(crate) output_tokens: Option, + pub(crate) reasoning_output_tokens: Option, + pub(crate) total_tokens: Option, pub(crate) duration_ms: Option, pub(crate) created_at: Option, pub(crate) completed_at: Option, diff --git a/codex-rs/analytics/src/reducer.rs b/codex-rs/analytics/src/reducer.rs index 930ae8bc58..c32d043c90 100644 --- a/codex-rs/analytics/src/reducer.rs +++ b/codex-rs/analytics/src/reducer.rs @@ -34,6 +34,7 @@ use codex_app_server_protocol::CodexErrorInfo; use codex_app_server_protocol::InitializeParams; use codex_app_server_protocol::RequestId; use codex_app_server_protocol::ServerNotification; +use codex_app_server_protocol::TokenUsageBreakdown; use codex_app_server_protocol::UserInput; use codex_git_utils::collect_git_info; use codex_git_utils::get_git_repo_root; @@ -83,6 +84,7 @@ struct TurnState { num_input_images: Option, resolved_config: Option, created_at: Option, + token_usage: Option, completed: Option, } @@ -206,6 +208,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, created_at: None, + token_usage: None, completed: None, }); turn_state.thread_id = Some(thread_id); @@ -358,6 +361,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, created_at: None, + token_usage: None, completed: None, }); turn_state.connection_id = Some(connection_id); @@ -382,10 +386,23 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, created_at: None, + token_usage: None, completed: None, }); turn_state.created_at = u64::try_from(notification.created_at).ok(); } + ServerNotification::ThreadTokenUsageUpdated(notification) => { + let turn_state = self.turns.entry(notification.turn_id).or_insert(TurnState { + connection_id: None, + thread_id: None, + num_input_images: None, + resolved_config: None, + created_at: None, + token_usage: None, + completed: None, + }); + turn_state.token_usage = Some(notification.token_usage.last); + } ServerNotification::TurnCompleted(notification) => { let turn_state = self.turns @@ -396,6 +413,7 @@ impl AnalyticsReducer { num_input_images: None, resolved_config: None, created_at: None, + token_usage: None, completed: None, }); turn_state.completed = Some(CompletedTurnState { @@ -509,6 +527,7 @@ fn codex_turn_event_params( personality, is_first_turn, } = resolved_config; + let token_usage = turn_state.token_usage.clone(); CodexTurnEventParams { thread_id, turn_id, @@ -540,6 +559,21 @@ fn codex_turn_event_params( subagent_tool_call_count: None, web_search_count: None, image_generation_count: None, + input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.input_tokens), + cached_input_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.cached_input_tokens), + output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.output_tokens), + reasoning_output_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.reasoning_output_tokens), + total_tokens: token_usage + .as_ref() + .map(|token_usage| token_usage.total_tokens), duration_ms: completed.duration_ms, created_at, completed_at: Some(completed.completed_at), diff --git a/codex-rs/app-server/src/bespoke_event_handling.rs b/codex-rs/app-server/src/bespoke_event_handling.rs index 04d3465f08..c381daefbf 100644 --- a/codex-rs/app-server/src/bespoke_event_handling.rs +++ b/codex-rs/app-server/src/bespoke_event_handling.rs @@ -1364,8 +1364,14 @@ pub(crate) async fn apply_bespoke_event_handling( .await; } EventMsg::TokenCount(token_count_event) => { - handle_token_count_event(conversation_id, event_turn_id, token_count_event, &outgoing) - .await; + handle_token_count_event( + conversation_id, + event_turn_id, + Some(&analytics_events_client), + token_count_event, + &outgoing, + ) + .await; } EventMsg::Error(ev) => { thread_watch_manager @@ -2208,6 +2214,7 @@ async fn handle_thread_rollback_failed( async fn handle_token_count_event( conversation_id: ThreadId, turn_id: String, + analytics_events_client: Option<&AnalyticsEventsClient>, token_count_event: TokenCountEvent, outgoing: &ThreadScopedOutgoingMessageSender, ) { @@ -2218,6 +2225,11 @@ async fn handle_token_count_event( turn_id, token_usage, }; + if let Some(analytics_events_client) = analytics_events_client { + analytics_events_client.track_notification( + ServerNotification::ThreadTokenUsageUpdated(notification.clone()), + ); + } outgoing .send_server_notification(ServerNotification::ThreadTokenUsageUpdated(notification)) .await; @@ -3551,6 +3563,7 @@ mod tests { handle_token_count_event( conversation_id, turn_id.clone(), + /*analytics_events_client*/ None, TokenCountEvent { info: Some(info), rate_limits: Some(rate_limits), @@ -3605,6 +3618,7 @@ mod tests { handle_token_count_event( conversation_id, turn_id.clone(), + /*analytics_events_client*/ None, TokenCountEvent { info: None, rate_limits: None, diff --git a/codex-rs/app-server/tests/suite/v2/turn_start.rs b/codex-rs/app-server/tests/suite/v2/turn_start.rs index 5b612791d2..a8552b32d9 100644 --- a/codex-rs/app-server/tests/suite/v2/turn_start.rs +++ b/codex-rs/app-server/tests/suite/v2/turn_start.rs @@ -309,6 +309,11 @@ async fn turn_start_tracks_turn_event_analytics() -> Result<()> { assert!(event["event_params"]["created_at"].as_u64().is_some()); assert!(event["event_params"]["completed_at"].as_u64().is_some()); assert!(event["event_params"]["duration_ms"].as_u64().is_some()); + assert_eq!(event["event_params"]["input_tokens"], 0); + assert_eq!(event["event_params"]["cached_input_tokens"], 0); + assert_eq!(event["event_params"]["output_tokens"], 0); + assert_eq!(event["event_params"]["reasoning_output_tokens"], 0); + assert_eq!(event["event_params"]["total_tokens"], 0); Ok(()) }