Trace tool call receipt, result readiness, and code-mode dispatch (#44661)

## Why

Tool results can be ready before the sampling loop collects them, and code-mode cells can outlive a turn. Separate trace milestones make these timings visible and link nested calls to the turn that dispatches them.

## What changed

- Emit `codex.tool_call_received` and `codex.tool_result_ready` events for direct and code-mode calls, using identifiers and tool names without arguments or output. Record readiness before result encoding or collection, including ordinary errors and aborted responses, but excluding fatal errors.
- Assign nested call IDs at broker receipt and carry them through dispatch. Trace dispatch and interruptions caused by cancellation, cell closure, or abandonment.
- Add code-mode execute and wait handler spans with conversation, turn, call, and cell identifiers and `completed`, `failed`, or `interrupted` outcomes.

GitOrigin-RevId: b098a29d8ee030c823665e4b1031a0d67cc09502
This commit is contained in:
Adam Perry @ OpenAI
2026-09-10 22:33:32 +00:00
committed by copyberry
parent e004dc6a4b
commit 84ed5744d9
13 changed files with 323 additions and 23 deletions
+47 -5
View File
@@ -8,6 +8,7 @@ use codex_code_mode::CodeModeSessionDelegate;
use codex_code_mode::NotificationFuture;
use codex_code_mode::ToolInvocationFuture;
use codex_protocol::ResponseItemId;
use codex_protocol::ThreadId;
use codex_protocol::models::FunctionCallOutputPayload;
use codex_protocol::models::ResponseItem;
use serde_json::Value as JsonValue;
@@ -19,12 +20,17 @@ use tracing::Instrument;
use super::ExecContext;
use super::PUBLIC_TOOL_NAME;
use super::submit_nested_tool;
use super::telemetry::DispatchInterruption;
use super::telemetry::NestedToolDispatchTrace;
use super::telemetry::trace_id;
use crate::session::step_context::StepContext;
use crate::tools::ExecutedToolCalls;
use crate::tools::call_trace;
use crate::tools::context::SharedTurnDiffTracker;
use crate::tools::parallel::ToolCallRuntime;
pub(super) struct CodeModeDispatchBroker {
thread_id: ThreadId,
dispatch_tx: async_channel::Sender<DispatchMessage>,
dispatch_rx: async_channel::Receiver<DispatchMessage>,
dispatch_gates: Arc<Mutex<HashMap<CellId, CellDispatchGate>>>,
@@ -38,9 +44,10 @@ struct CellDispatchGate {
}
impl CodeModeDispatchBroker {
pub(super) fn new(executed_tool_calls: ExecutedToolCalls) -> Self {
pub(super) fn new(thread_id: ThreadId, executed_tool_calls: ExecutedToolCalls) -> Self {
let (dispatch_tx, dispatch_rx) = async_channel::unbounded();
Self {
thread_id,
dispatch_tx,
dispatch_rx,
dispatch_gates: Arc::new(Mutex::new(HashMap::new())),
@@ -141,9 +148,9 @@ impl CodeModeDispatchBroker {
}
DispatchMessage::InvokeTool {
invocation,
mut dispatch_trace,
cancellation_token,
response_tx,
span,
} => {
let cell_id = invocation.cell_id.clone();
if !wait_until_cell_ready_for_dispatch(
@@ -153,11 +160,18 @@ impl CodeModeDispatchBroker {
)
.await
{
let outcome = if cancellation_token.is_cancelled() {
DispatchInterruption::Cancelled
} else {
DispatchInterruption::CellClosed
};
dispatch_trace.interruption = Some(outcome);
remove_dispatch_gate(&dispatch_gates, &cell_id);
continue;
}
let host = Arc::clone(&host);
let dispatch_gates = Arc::clone(&dispatch_gates);
let span = dispatch_trace.span.clone();
tokio::spawn(async move {
let invocation = {
let dispatch_gate = track_completeness.then(|| {
@@ -169,11 +183,22 @@ impl CodeModeDispatchBroker {
cancellation_token.is_cancelled()
|| !gates.contains_key(&cell_id)
}) {
let outcome = if cancellation_token.is_cancelled() {
DispatchInterruption::Cancelled
} else {
DispatchInterruption::CellClosed
};
dispatch_trace.interruption = Some(outcome);
return;
}
// Submission and cell closure share this gate.
span.in_scope(|| {
host.submit_tool(invocation, cancellation_token.clone())
dispatch_trace.interruption = None;
host.submit_tool(
invocation,
dispatch_trace.call_id.clone(),
cancellation_token.clone(),
)
})
.instrument(span)
};
@@ -254,6 +279,7 @@ impl CodeModeSessionDelegate for CodeModeDispatchBroker {
level = "info",
skip_all,
fields(
conversation.id = %self.thread_id,
cell.id = %invocation.cell_id,
runtime_tool_call_id = invocation.runtime_tool_call_id.as_str(),
tool_name = invocation.tool_name.name.as_str(),
@@ -266,16 +292,30 @@ impl CodeModeSessionDelegate for CodeModeDispatchBroker {
cancellation_token: CancellationToken,
) -> ToolInvocationFuture<'a> {
Box::pin(async move {
let call_id = format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4());
call_trace::received(
self.thread_id,
&invocation.tool_name,
&call_id,
call_trace::Receipt::CodeModeBroker {
cell_id: trace_id(invocation.cell_id.as_str()),
runtime_tool_call_id: trace_id(&invocation.runtime_tool_call_id),
},
);
let mut dispatch_trace =
Box::new(NestedToolDispatchTrace::new(self.thread_id, call_id));
if cancellation_token.is_cancelled() {
dispatch_trace.interruption = Some(DispatchInterruption::Cancelled);
return Err("code mode nested tool call cancelled".to_string());
}
let (response_tx, response_rx) = oneshot::channel();
// Only the worker can tell whether dispatch beats cancellation once the call is queued.
self.dispatch_tx
.send(DispatchMessage::InvokeTool {
invocation,
dispatch_trace,
cancellation_token: cancellation_token.clone(),
response_tx,
span: tracing::Span::current(),
})
.await
.map_err(|_| "code mode nested tool dispatcher is unavailable".to_string())?;
@@ -329,9 +369,9 @@ impl CodeModeSessionDelegate for CodeModeDispatchBroker {
enum DispatchMessage {
InvokeTool {
invocation: CodeModeNestedToolCall,
dispatch_trace: Box<NestedToolDispatchTrace>,
cancellation_token: CancellationToken,
response_tx: oneshot::Sender<Result<JsonValue, String>>,
span: tracing::Span,
},
Notify {
call_id: String,
@@ -363,12 +403,14 @@ impl CoreTurnHost {
fn submit_tool(
&self,
invocation: CodeModeNestedToolCall,
call_id: String,
cancellation_token: CancellationToken,
) -> impl std::future::Future<Output = Result<JsonValue, String>> + Send + 'static {
let invocation = submit_nested_tool(
self.exec.clone(),
self.tool_runtime.clone(),
invocation,
call_id,
cancellation_token,
)
.map_err(|error| error.to_string());
@@ -116,6 +116,7 @@ async fn detached_code_mode_callback_keeps_thread_id_on_dispatch_span() -> anyho
));
let step = StepContext::for_test(Arc::clone(&turn)).with_tool_router_for_test(router);
let broker = Arc::new(CodeModeDispatchBroker::new(
session.thread_id,
/*executed_tool_calls*/ Default::default(),
));
let cell_id = CellId::new("audit-cell".to_string());
@@ -14,6 +14,7 @@ use super::PUBLIC_TOOL_NAME;
use super::handle_runtime_response;
use super::is_exec_tool_name;
use super::telemetry::CodeModeToolCallGuard;
use super::telemetry::trace_id;
type CodeModeNestedTool = (Arc<ToolSpec>, Option<Arc<dyn CoreToolRuntime>>);
@@ -80,6 +81,7 @@ impl CodeModeExecuteHandler {
.await
.map_err(FunctionCallError::RespondToModel)?;
let cell_id = started_cell.cell_id.clone();
tracing::Span::current().record("cell.id", trace_id(cell_id.as_str()));
telemetry.cell_id = Some(cell_id.to_string());
exec.session
.services
@@ -170,10 +172,25 @@ impl ToolExecutor<ToolInvocation> for CodeModeExecuteHandler {
}
impl CodeModeExecuteHandler {
// Default to interrupted if this future is dropped; telemetry::CodeModeToolCallGuard::finish
// overwrites this handler's captured span on explicit success or failure.
#[tracing::instrument(
name = "code_mode.handler.execute",
level = "info",
skip_all,
fields(
conversation.id = %invocation.session.thread_id,
turn_id = invocation.turn.sub_id.as_str(),
call_id = trace_id(&invocation.call_id),
cell.id = tracing::field::Empty,
outcome = "interrupted",
)
)]
async fn handle_call(
&self,
invocation: ToolInvocation,
) -> Result<Box<dyn crate::tools::context::ToolOutput>, FunctionCallError> {
let handler_span = tracing::Span::current();
let originating_item_id = invocation.originating_item_id().await;
let ToolInvocation {
session,
@@ -192,6 +209,7 @@ impl CodeModeExecuteHandler {
turn.turn_metadata_state.clone(),
call_id.clone(),
PUBLIC_TOOL_NAME,
handler_span,
);
let result = match payload {
ToolPayload::Custom { input } if is_exec_tool_name(&tool_name) => self
+38 -11
View File
@@ -17,6 +17,7 @@ use codex_code_mode::CodeModeSession;
use codex_code_mode::CodeModeSessionProvider;
use codex_code_mode::CodeModeToolKind;
use codex_code_mode::RuntimeResponse;
use codex_protocol::ThreadId;
use codex_protocol::models::FunctionCallOutputContentItem;
use futures::future::join_all;
use serde_json::Value as JsonValue;
@@ -31,6 +32,7 @@ use crate::session::session::Session;
use crate::session::step_context::StepContext;
use crate::session::turn_context::TurnContext;
use crate::tools::ExecutedToolCalls;
use crate::tools::call_trace;
use crate::tools::context::FunctionToolOutput;
use crate::tools::context::SharedTurnDiffTracker;
use crate::tools::context::ToolPayload;
@@ -78,11 +80,12 @@ pub(crate) struct CodeModeService {
impl CodeModeService {
pub(crate) fn new(
thread_id: ThreadId,
session_provider: Arc<dyn CodeModeSessionProvider>,
config: &CodeModeConfig,
executed_tool_calls: ExecutedToolCalls,
) -> Self {
let dispatch_broker = Arc::new(CodeModeDispatchBroker::new(executed_tool_calls));
let dispatch_broker = Arc::new(CodeModeDispatchBroker::new(thread_id, executed_tool_calls));
let availability = session_provider.availability();
Self {
session: OnceCell::new(),
@@ -343,6 +346,7 @@ fn submit_nested_tool(
exec: ExecContext,
tool_runtime: ToolCallRuntime,
invocation: CodeModeNestedToolCall,
call_id: String,
cancellation_token: CancellationToken,
) -> Result<
impl std::future::Future<Output = Result<JsonValue, FunctionCallError>> + Send + 'static,
@@ -355,20 +359,43 @@ fn submit_nested_tool(
tool_kind,
input,
} = invocation;
if is_exec_tool_name(&tool_name) {
return Err(FunctionCallError::RespondToModel(format!(
"{PUBLIC_TOOL_NAME} cannot invoke itself"
)));
}
let payload = match build_nested_tool_payload(tool_kind, &tool_name, input) {
let thread_id = exec.session.thread_id;
let turn_id = exec.turn.sub_id.clone();
let tool_name = tool_name.with_default_namespace();
// A cell can outlive a turn; the broker records arrival before a dispatching turn is known.
tracing::event!(
name: "codex.code_mode.nested_tool_dispatched",
target: "codex_otel.trace_safe",
tracing::Level::INFO,
event.name = "codex.code_mode.nested_tool_dispatched",
conversation.id = %thread_id,
turn_id = turn_id.as_str(),
cell.id = telemetry::trace_id(cell_id.as_str()),
runtime_tool_call_id = telemetry::trace_id(&runtime_tool_call_id),
call_id = call_id.as_str(),
);
let payload = if is_exec_tool_name(&tool_name) {
Err(format!("{PUBLIC_TOOL_NAME} cannot invoke itself"))
} else {
build_nested_tool_payload(tool_kind, &tool_name, input)
};
let payload = match payload {
Ok(payload) => payload,
Err(error) => return Err(FunctionCallError::RespondToModel(error)),
Err(error) => {
call_trace::result_ready(
thread_id,
&turn_id,
&tool_name,
&call_id,
call_trace::Source::CodeMode,
);
return Err(FunctionCallError::RespondToModel(error));
}
};
let call = ToolCall {
tool_name: tool_name.with_default_namespace(),
call_id: format!("{PUBLIC_TOOL_NAME}-{}", uuid::Uuid::new_v4()),
tool_name,
call_id,
payload,
encrypted_function_args: None,
};
@@ -1,9 +1,13 @@
//! Code-mode tool analytics and tracing for handlers and queued nested dispatch.
use codex_analytics::AnalyticsEventsClient;
use codex_analytics::CodeModeToolCallFact;
use codex_analytics::CodeModeToolCallStatus;
use codex_analytics::TurnAnalyticsMetadata;
use codex_protocol::ThreadId;
use std::sync::Arc;
use std::time::Duration;
use tracing::Span;
pub(super) struct CodeModeToolCallGuard {
analytics: AnalyticsEventsClient,
@@ -15,6 +19,7 @@ pub(super) struct CodeModeToolCallGuard {
tool_name: &'static str,
started_at_ms: u64,
status: CodeModeToolCallStatus,
handler_span: Span,
}
impl CodeModeToolCallGuard {
@@ -25,6 +30,7 @@ impl CodeModeToolCallGuard {
turn_metadata: Arc<dyn TurnAnalyticsMetadata>,
call_id: String,
tool_name: &'static str,
handler_span: Span,
) -> Self {
Self {
analytics,
@@ -36,15 +42,19 @@ impl CodeModeToolCallGuard {
tool_name,
started_at_ms: codex_analytics::now_unix_millis(),
status: CodeModeToolCallStatus::Interrupted,
handler_span,
}
}
pub(super) fn finish(&mut self, success: bool) {
let mut outcome = "failed";
self.status = if success {
outcome = "completed";
CodeModeToolCallStatus::Completed
} else {
CodeModeToolCallStatus::Failed
};
self.handler_span.record("outcome", outcome);
}
pub(super) fn record_code_mode_host_duration(&self, duration: Duration) {
@@ -85,3 +95,54 @@ impl Drop for CodeModeToolCallGuard {
});
}
}
pub(super) struct NestedToolDispatchTrace {
thread_id: ThreadId,
pub(super) call_id: String,
pub(super) interruption: Option<DispatchInterruption>,
pub(super) span: Span,
}
impl NestedToolDispatchTrace {
pub(super) fn new(thread_id: ThreadId, call_id: String) -> Self {
Self {
thread_id,
call_id,
interruption: Some(DispatchInterruption::Abandoned),
span: Span::current(),
}
}
}
impl Drop for NestedToolDispatchTrace {
fn drop(&mut self) {
let Some(outcome) = self.interruption.take() else {
return;
};
let outcome = match outcome {
DispatchInterruption::Cancelled => "cancelled",
DispatchInterruption::CellClosed => "cell_closed",
DispatchInterruption::Abandoned => "abandoned",
};
tracing::event!(
name: "codex.code_mode.nested_tool_dispatch_interrupted",
target: "codex_otel.trace_safe",
parent: &self.span,
tracing::Level::INFO,
event.name = "codex.code_mode.nested_tool_dispatch_interrupted",
conversation.id = %self.thread_id,
call_id = self.call_id.as_str(),
outcome,
);
}
}
pub(super) enum DispatchInterruption {
Cancelled,
CellClosed,
Abandoned,
}
pub(super) fn trace_id(id: &str) -> Option<&str> {
(!id.is_empty() && id.len() <= 256).then_some(id)
}
@@ -17,6 +17,7 @@ use super::ExecContext;
use super::WAIT_TOOL_NAME;
use super::handle_runtime_response;
use super::telemetry::CodeModeToolCallGuard;
use super::telemetry::trace_id;
use super::wait_spec::create_wait_tool;
pub struct CodeModeWaitHandler;
@@ -63,10 +64,25 @@ impl ToolExecutor<ToolInvocation> for CodeModeWaitHandler {
}
impl CodeModeWaitHandler {
// Default to interrupted if this future is dropped; telemetry::CodeModeToolCallGuard::finish
// overwrites this handler's captured span on explicit success or failure, including early errors.
#[tracing::instrument(
name = "code_mode.handler.wait",
level = "info",
skip_all,
fields(
conversation.id = %invocation.session.thread_id,
turn_id = invocation.turn.sub_id.as_str(),
call_id = trace_id(&invocation.call_id),
cell.id = tracing::field::Empty,
outcome = "interrupted",
)
)]
async fn handle_call(
&self,
invocation: ToolInvocation,
) -> Result<Box<dyn crate::tools::context::ToolOutput>, FunctionCallError> {
let handler_span = tracing::Span::current();
let ToolInvocation {
session,
turn,
@@ -84,6 +100,7 @@ impl CodeModeWaitHandler {
turn.turn_metadata_state.clone(),
call_id.clone(),
WAIT_TOOL_NAME,
handler_span,
);
let result = match payload {
ToolPayload::Function { arguments }
@@ -123,6 +140,7 @@ impl CodeModeWaitHandler {
| codex_code_mode::RuntimeResponse::Terminated { cell_id, .. }
| codex_code_mode::RuntimeResponse::Result { cell_id, .. } => cell_id,
};
tracing::Span::current().record("cell.id", trace_id(runtime_cell_id.as_str()));
telemetry.cell_id = Some(runtime_cell_id.to_string());
exec.session
.services