openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-11-08 20:40:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Correlate code mode tool analytics with model responses (#36729)

Browse Source 
## What changed

- Emit dynamic tool-call analytics for code mode `exec` and `wait` calls, including duration and terminal status.
- Associate code-mode cells and their nested tool calls with the originating and subsequent model response IDs.
- Buffer correlated tool events until the next response is known, while flushing pending events when turns, threads, or the analytics queue close.

## Testing

- Add reducer coverage for correlating `exec`, `wait`, and nested tool events across turns.
- Add an app-server test that verifies production analytics include the cell and response IDs for a code-mode `exec` call.

GitOrigin-RevId: 8b34ab59c1ad68ebbd57f9c8a05f19ab04df48a4
This commit is contained in:
jif
2026-08-03 10:19:04 +00:00
committed by copyberry
parent dae2122214
commit 155f1ca9e5
13 changed files with 797 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
codex-rs/core/src/tools/code_mode/execute_handler.rs
+36 -3
View File
@@ -12,6 +12,7 @@ use super::ExecContext;
use super::PUBLIC_TOOL_NAME;
use super::handle_runtime_response;
use super::is_exec_tool_name;
use super::telemetry::CodeModeToolCallGuard;
pub struct CodeModeExecuteHandler {
spec: ToolSpec,
@@ -32,6 +33,7 @@ impl CodeModeExecuteHandler {
turn: std::sync::Arc<crate::session::turn_context::TurnContext>,
call_id: String,
code: String,
telemetry: &mut CodeModeToolCallGuard,
) -> Result<FunctionToolOutput, FunctionCallError> {
let args =
codex_code_mode::parse_exec_source(&code).map_err(FunctionCallError::RespondToModel)?;
@@ -53,6 +55,16 @@ impl CodeModeExecuteHandler {
.await
.map_err(FunctionCallError::RespondToModel)?;
let cell_id = started_cell.cell_id.clone();
telemetry.cell_id = Some(cell_id.to_string());
exec.session
.services
.analytics_events_client
.track_code_mode_tool_call(codex_analytics::CodeModeToolCallFact::CellStarted {
thread_id: exec.session.thread_id.to_string(),
turn_id: exec.turn.sub_id.clone(),
call_id: call_id.clone(),
cell_id: cell_id.to_string(),
});
if let Some(executed_tool_calls) = exec.session.services.executed_tool_calls.as_ref() {
executed_tool_calls.register_cell(&cell_id, &call_id);
}
@@ -87,6 +99,14 @@ impl CodeModeExecuteHandler {
.services
.code_mode_service
.finish_cell_dispatch(&cell_id);
exec.session
.services
.analytics_events_client
.track_code_mode_tool_call(codex_analytics::CodeModeToolCallFact::CellClosed {
thread_id: exec.session.thread_id.to_string(),
turn_id: exec.turn.sub_id.clone(),
cell_id: cell_id.to_string(),
});
}
exec.session.services.elicitations.wait_until_clear().await;
handle_runtime_response(&exec, response, args.max_output_tokens, started_at)
@@ -123,15 +143,28 @@ impl CodeModeExecuteHandler {
..
} = invocation;
match payload {
let mut telemetry = CodeModeToolCallGuard::new(
session.services.analytics_events_client.clone(),
session.thread_id.to_string(),
turn.sub_id.clone(),
call_id.clone(),
PUBLIC_TOOL_NAME,
);
let result = match payload {
ToolPayload::Custom { input } if is_exec_tool_name(&tool_name) => self
.execute(session, turn, call_id, input)
.execute(session, turn, call_id, input, &mut telemetry)
.await
.map(boxed_tool_output),
_ => Err(FunctionCallError::RespondToModel(format!(
"{PUBLIC_TOOL_NAME} expects raw JavaScript source text"
))),
}
};
telemetry.finish(
result
.as_ref()
.is_ok_and(|output| output.success_for_logging()),
);
result
}
}
codex-rs/core/src/tools/code_mode/mod.rs
+11 -1
View File
@@ -2,6 +2,7 @@ mod delegate;
mod execute_handler;
pub(crate) mod execute_spec;
mod response_adapter;
mod telemetry;
mod wait_handler;
pub(crate) mod wait_spec;
@@ -314,7 +315,7 @@ fn truncate_code_mode_result(
}
async fn call_nested_tool(
_exec: ExecContext,
exec: ExecContext,
tool_runtime: ToolCallRuntime,
invocation: CodeModeNestedToolCall,
cancellation_token: CancellationToken,
@@ -343,6 +344,15 @@ async fn call_nested_tool(
payload,
encrypted_function_args: None,
};
exec.session
.services
.analytics_events_client
.track_code_mode_tool_call(codex_analytics::CodeModeToolCallFact::ChildStarted {
thread_id: exec.session.thread_id.to_string(),
turn_id: exec.turn.sub_id.clone(),
call_id: call.call_id.clone(),
cell_id: cell_id.to_string(),
});
let result = tool_runtime
.handle_tool_call_with_source(
call,
codex-rs/core/src/tools/code_mode/telemetry.rs
+59
View File
@@ -0,0 +1,59 @@
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::CodeModeToolCallFact;
use codex_analytics::CodeModeToolCallStatus;
pub(super) struct CodeModeToolCallGuard {
analytics: AnalyticsEventsClient,
thread_id: String,
turn_id: String,
call_id: String,
pub(super) cell_id: Option<String>,
tool_name: &'static str,
started_at_ms: u64,
status: CodeModeToolCallStatus,
}
impl CodeModeToolCallGuard {
pub(super) fn new(
analytics: AnalyticsEventsClient,
thread_id: String,
turn_id: String,
call_id: String,
tool_name: &'static str,
) -> Self {
Self {
analytics,
thread_id,
turn_id,
call_id,
cell_id: None,
tool_name,
started_at_ms: codex_analytics::now_unix_millis(),
status: CodeModeToolCallStatus::Interrupted,
}
}
pub(super) fn finish(&mut self, success: bool) {
self.status = if success {
CodeModeToolCallStatus::Completed
} else {
CodeModeToolCallStatus::Failed
};
}
}
impl Drop for CodeModeToolCallGuard {
fn drop(&mut self) {
self.analytics
.track_code_mode_tool_call(CodeModeToolCallFact::Completed {
thread_id: self.thread_id.clone(),
turn_id: self.turn_id.clone(),
call_id: self.call_id.clone(),
cell_id: self.cell_id.clone(),
tool_name: self.tool_name.to_string(),
started_at_ms: self.started_at_ms,
completed_at_ms: codex_analytics::now_unix_millis(),
status: self.status,
});
}
}
codex-rs/core/src/tools/code_mode/wait_handler.rs
+55 -28
View File
@@ -16,6 +16,7 @@ use super::DEFAULT_WAIT_YIELD_TIME_MS;
use super::ExecContext;
use super::WAIT_TOOL_NAME;
use super::handle_runtime_response;
use super::telemetry::CodeModeToolCallGuard;
use super::wait_spec::create_wait_tool;
pub struct CodeModeWaitHandler;
@@ -72,19 +73,23 @@ impl CodeModeWaitHandler {
..
} = invocation;
match payload {
let mut telemetry = CodeModeToolCallGuard::new(
session.services.analytics_events_client.clone(),
session.thread_id.to_string(),
turn.sub_id.clone(),
call_id.clone(),
WAIT_TOOL_NAME,
);
let result = match payload {
ToolPayload::Function { arguments }
if tool_name.namespace.is_none() && tool_name.name.as_str() == WAIT_TOOL_NAME =>
{
let args: ExecWaitArgs = parse_arguments(&arguments)?;
let args: ExecWaitArgs = parse_arguments(&arguments).inspect_err(|_error| {
telemetry.finish(/*success*/ false);
})?;
let exec = ExecContext { session, turn };
let started_at = std::time::Instant::now();
let cell_id = codex_code_mode::CellId::new(args.cell_id);
if let Some(executed_tool_calls) =
exec.session.services.executed_tool_calls.as_ref()
{
executed_tool_calls.register_cell(&cell_id, &call_id);
}
let wait_response = if args.terminate {
exec.session
.services
@@ -101,41 +106,63 @@ impl CodeModeWaitHandler {
})
.await
}
.map_err(FunctionCallError::RespondToModel)?;
if let codex_code_mode::WaitOutcome::LiveCell(response) = &wait_response
&& !matches!(response, codex_code_mode::RuntimeResponse::Yielded { .. })
{
// Only a live-cell wait can close a CodeCell. A missing
// cell is still an ordinary `wait` tool result, but there
// is no runtime object for the reducer to complete.
.map_err(|error| {
telemetry.finish(/*success*/ false);
FunctionCallError::RespondToModel(error)
})?;
if let codex_code_mode::WaitOutcome::LiveCell(response) = &wait_response {
let runtime_cell_id = match response {
codex_code_mode::RuntimeResponse::Yielded { cell_id, .. }
| codex_code_mode::RuntimeResponse::Terminated { cell_id, .. }
| codex_code_mode::RuntimeResponse::Result { cell_id, .. } => cell_id,
};
exec.session
.services
.rollout_thread_trace
.code_cell_trace_context(
exec.turn.sub_id.as_str(),
runtime_cell_id.as_str(),
)
.record_ended(response);
exec.session
.services
.code_mode_service
.finish_cell_dispatch(runtime_cell_id);
telemetry.cell_id = Some(runtime_cell_id.to_string());
if let Some(executed_tool_calls) =
exec.session.services.executed_tool_calls.as_ref()
{
executed_tool_calls.register_cell(runtime_cell_id, &call_id);
}
if !matches!(response, codex_code_mode::RuntimeResponse::Yielded { .. }) {
exec.session
.services
.rollout_thread_trace
.code_cell_trace_context(
exec.turn.sub_id.as_str(),
runtime_cell_id.as_str(),
)
.record_ended(response);
exec.session
.services
.code_mode_service
.finish_cell_dispatch(runtime_cell_id);
exec.session
.services
.analytics_events_client
.track_code_mode_tool_call(
codex_analytics::CodeModeToolCallFact::CellClosed {
thread_id: exec.session.thread_id.to_string(),
turn_id: exec.turn.sub_id.clone(),
cell_id: runtime_cell_id.to_string(),
},
);
}
}
exec.session.services.elicitations.wait_until_clear().await;
handle_runtime_response(&exec, wait_response.into(), args.max_tokens, started_at)
.await
.map(boxed_tool_output)
.map_err(FunctionCallError::RespondToModel)
.map(boxed_tool_output)
}
_ => Err(FunctionCallError::RespondToModel(format!(
"{WAIT_TOOL_NAME} expects JSON arguments"
))),
}
};
telemetry.finish(
result
.as_ref()
.is_ok_and(codex_tools::ToolOutput::success_for_logging),
);
result
}
}