//! Thread-scoped rollout trace helpers. //! //! A rollout bundle can contain a root thread plus spawned child threads. This //! context owns the stable identity for one thread inside that bundle. Keeping //! thread-local event methods here avoids repeatedly plumbing `thread_id` //! through session code. use codex_protocol::mcp::CallToolResult; use codex_protocol::models::ResponseItem; use codex_protocol::protocol::AgentStatus; use codex_protocol::protocol::EventMsg; use codex_protocol::protocol::SessionSource; use serde::Serialize; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use tracing::debug; use tracing::warn; use uuid::Uuid; use crate::AgentThreadId; use crate::CodeCellTraceContext; use crate::CodexTurnId; use crate::CompactionId; use crate::CompactionTraceContext; use crate::ExecutionStatus; use crate::InferenceTraceContext; use crate::McpCallTraceContext; use crate::RawPayloadKind; use crate::RawTraceEventContext; use crate::RawTraceEventPayload; use crate::RolloutStatus; use crate::ToolCallId; use crate::ToolDispatchInvocation; use crate::ToolDispatchTraceContext; use crate::TraceCaptureLimits; use crate::TraceWriter; use crate::protocol_event::codex_turn_trace_event; use crate::protocol_event::tool_runtime_trace_event; use crate::protocol_event::wrapped_protocol_event_type; /// Environment variable that enables local trace-bundle recording. /// /// The value is a root directory. Each independent root session gets one child /// bundle directory. Spawned child threads share their root session's bundle so /// one reduced `state.json` describes the whole multi-agent rollout tree. pub const CODEX_ROLLOUT_TRACE_ROOT_ENV: &str = "CODEX_ROLLOUT_TRACE_ROOT"; /// Metadata captured once at thread/session start. /// /// This payload is intentionally operational rather than reduced: it is a raw /// payload that later reducers can mine as the reduced thread model evolves. #[derive(Serialize)] pub struct ThreadStartedTraceMetadata { pub thread_id: String, pub agent_path: String, pub task_name: Option, pub nickname: Option, pub agent_role: Option, pub session_source: SessionSource, pub cwd: std::path::PathBuf, pub rollout_path: Option, pub model: String, pub provider_name: String, pub approval_policy: String, pub permissions_profile: String, } /// Trace-only payload for a child completion notification delivered to its parent. #[derive(Serialize)] pub struct AgentResultTracePayload<'a> { pub child_agent_path: &'a str, pub message: &'a str, pub status: &'a AgentStatus, } /// No-op capable trace handle for one thread in a rollout bundle. #[derive(Clone, Debug)] pub struct ThreadTraceContext { state: ThreadTraceContextState, } #[derive(Clone, Debug)] enum ThreadTraceContextState { Disabled, Enabled(EnabledThreadTraceContext), } #[derive(Clone, Debug)] struct EnabledThreadTraceContext { writer: Arc, root_thread_id: AgentThreadId, thread_id: AgentThreadId, } impl ThreadTraceContext { /// Builds a context that accepts trace calls and records nothing. pub fn disabled() -> Self { Self { state: ThreadTraceContextState::Disabled, } } /// Starts tracing when configured; an enabled recorder must initialize successfully. pub fn start_root( metadata: ThreadStartedTraceMetadata, limits: TraceCaptureLimits, ) -> anyhow::Result { let Some(root) = std::env::var_os(CODEX_ROLLOUT_TRACE_ROOT_ENV) else { return Ok(Self::disabled()); }; start_root_in_root(&PathBuf::from(root), metadata, limits) } /// Starts a root trace in a known directory. /// /// This is public for tests that need replayable trace bundles without /// mutating process environment. pub fn start_root_in_root_for_test( root: &Path, metadata: ThreadStartedTraceMetadata, ) -> anyhow::Result { start_root_in_root(root, metadata, TraceCaptureLimits::default()) } /// Starts one thread lifecycle inside an existing rollout bundle. pub(crate) fn start( writer: Arc, root_thread_id: AgentThreadId, metadata: ThreadStartedTraceMetadata, ) -> anyhow::Result { let context = EnabledThreadTraceContext { writer, root_thread_id, thread_id: metadata.thread_id.clone(), }; // Thread creation reports storage failures before returning the new handle. let metadata_payload = context .writer .write_json_payload(RawPayloadKind::SessionMetadata, &metadata)?; context.writer.append(RawTraceEventPayload::ThreadStarted { thread_id: metadata.thread_id, agent_path: metadata.agent_path, metadata_payload: Some(metadata_payload), })?; Ok(Self { state: ThreadTraceContextState::Enabled(context), }) } /// Publishes queued evidence before reading a snapshot of a running bundle. pub fn flush(&self) -> anyhow::Result<()> { match &self.state { ThreadTraceContextState::Disabled => Ok(()), ThreadTraceContextState::Enabled(context) => context.writer.flush(), } } /// Returns whether this handle will write trace events. /// /// Most methods have their own disabled fast path. Callers should branch on /// this only when preparing trace payloads would otherwise clone data the /// production path needs to move elsewhere. pub fn is_enabled(&self) -> bool { matches!(self.state, ThreadTraceContextState::Enabled(_)) } /// Starts a new child thread in this context's rollout tree. pub fn start_child_thread_trace( &self, metadata: ThreadStartedTraceMetadata, ) -> anyhow::Result { match &self.state { ThreadTraceContextState::Disabled => Ok(Self::disabled()), ThreadTraceContextState::Enabled(context) => Self::start( Arc::clone(&context.writer), context.root_thread_id.clone(), metadata, ), } } /// Reattaches an evicted child to this rollout without starting a new thread. pub fn resume_child_thread_trace(&self, thread_id: String) -> anyhow::Result { let ThreadTraceContextState::Enabled(parent) = &self.state else { return Ok(Self::disabled()); }; let context = EnabledThreadTraceContext { writer: Arc::clone(&parent.writer), root_thread_id: parent.root_thread_id.clone(), thread_id: thread_id.clone(), }; context .writer .append(RawTraceEventPayload::ThreadResumed { thread_id })?; Ok(Self { state: ThreadTraceContextState::Enabled(context), }) } /// Emits terminal trace events when the thread stops. /// /// Spawned child sessions share their root bundle, so only the root /// thread end closes the rollout. This is not recording finalization: child /// threads and detached producers may still hold the shared writer. /// Recording errors stay with the writer and prevent the closing event. pub fn record_ended(&self, status: RolloutStatus) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; context.append_best_effort(RawTraceEventPayload::ThreadEnded { thread_id: context.thread_id.clone(), status: status.clone(), }); if context.thread_id == context.root_thread_id { context.append_best_effort(RawTraceEventPayload::RolloutEnded { status }); } } /// Wraps selected protocol events as raw trace breadcrumbs. /// /// High-volume stream deltas stay out of this wrapper; typed inference, /// tool, terminal, and code-mode hooks provide the canonical runtime data. pub fn record_protocol_event(&self, event: &EventMsg) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; let Some(event_type) = wrapped_protocol_event_type(event) else { return; }; let event_type = event_type.to_owned(); let _ = context.writer.event_with_payload( RawTraceEventContext::default(), RawPayloadKind::ProtocolEvent, event, move |event_payload| RawTraceEventPayload::ProtocolEventObserved { event_type, event_payload, }, ); } /// Records turn lifecycle and history acceptance from existing protocol events. pub fn record_codex_turn_event(&self, default_turn_id: &str, event: &EventMsg) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; if let EventMsg::RawResponseItem(event) = event { let Some(item_id) = event.item.id().map(ToString::to_string) else { return; }; let accepted = move |agent_message_payload| RawTraceEventPayload::HistoryItemAccepted { item_id, agent_message_payload, }; let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id); if matches!(event.item, ResponseItem::AgentMessage { .. }) { // Publish the receipt and its history event together, in worker order. let _ = context.writer.event_with_payload( turn, RawPayloadKind::ProtocolEvent, &event.item, move |payload| accepted(Some(payload)), ); } else { let _ = context.writer.enqueue_event(turn, accepted(None)); } return; } let Some(trace_event) = codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event) else { return; }; context.append_with_context_best_effort( trace_event.context_turn_id.clone(), trace_event.payload, ); } /// Emits typed runtime tool events from existing protocol lifecycle events. /// /// These events are runtime observations on an already-dispatched tool. The /// dispatch trace records the caller-facing boundary; these payloads explain /// what Codex did while executing that boundary. pub fn record_tool_call_event(&self, codex_turn_id: impl Into, event: &EventMsg) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; let Some(trace_event) = tool_runtime_trace_event(event) else { return; }; let envelope = RawTraceEventContext::turn(&context.thread_id, &codex_turn_id.into()); match trace_event { crate::protocol_event::ToolRuntimeTraceEvent::Started { tool_call_id, kind, payload, } => { let tool_call_id = tool_call_id.to_owned(); let _ = context.writer.event_with_payload( envelope, RawPayloadKind::ToolRuntimeEvent, &payload, move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeStarted { tool_call_id, kind, runtime_payload, }, ); } crate::protocol_event::ToolRuntimeTraceEvent::Ended { tool_call_id, status, payload, } => { let tool_call_id = tool_call_id.to_owned(); let _ = context.writer.event_with_payload( envelope, RawPayloadKind::ToolRuntimeEvent, &payload, move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeEnded { tool_call_id, status, runtime_payload, }, ); } } } /// Records the full MCP backend result before callbacks, sanitization, or UI truncation. /// The dispatch trace separately records the result returned to the caller. pub fn record_mcp_tool_result( &self, codex_turn_id: &str, tool_call_id: &str, result: &Result, ) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; #[derive(Serialize)] struct Payload<'a> { result: &'a Result, } let tool_call_id = tool_call_id.to_owned(); let status = if result.is_ok() { ExecutionStatus::Completed } else { ExecutionStatus::Failed }; let _ = context.writer.event_with_payload( RawTraceEventContext::turn(&context.thread_id, codex_turn_id), RawPayloadKind::ToolRuntimeEvent, &Payload { result }, move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeEnded { tool_call_id, status, runtime_payload, }, ); } /// Records the exec actually selected by stdin, since its process handle can be reused. pub fn record_stdin_target( &self, codex_turn_id: &str, tool_call_id: &str, exec_tool_call_id: &str, ) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; context.append_with_context_best_effort( codex_turn_id.to_string(), RawTraceEventPayload::StdinTargetResolved { tool_call_id: tool_call_id.to_string(), exec_tool_call_id: exec_tool_call_id.to_string(), }, ); } /// Emits the v2 child-to-parent completion message as an explicit graph edge. /// /// The notification is runtime delivery from a completed child turn into /// the parent's mailbox, not a tool call executed by the child. Recording it /// directly preserves timing and source without making the reducer infer /// the edge from a later parent prompt snapshot. pub fn record_agent_result_interaction( &self, child_codex_turn_id: impl Into, parent_thread_id: impl Into, payload: &AgentResultTracePayload<'_>, ) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; let child_codex_turn_id = child_codex_turn_id.into(); let parent_thread_id = parent_thread_id.into(); let child_thread_id = context.thread_id.clone(); let message = payload.message.to_owned(); let edge_id = format!("edge:agent_result:{child_thread_id}:{child_codex_turn_id}:{parent_thread_id}"); let _ = context.writer.event_with_payload( RawTraceEventContext::turn(&child_thread_id, &child_codex_turn_id), RawPayloadKind::AgentResult, payload, move |reference| RawTraceEventPayload::AgentResultObserved { edge_id, child_thread_id, child_codex_turn_id, parent_thread_id, message, carried_payload: Some(reference), }, ); } /// Emits a turn-start lifecycle event. /// /// Most production turn lifecycle wiring lives outside this PR layer, but /// trace-focused integration tests need a small explicit hook so reducer /// inputs remain valid without exercising the full session loop. pub fn record_codex_turn_started(&self, codex_turn_id: impl Into) { let ThreadTraceContextState::Enabled(context) = &self.state else { return; }; let codex_turn_id = codex_turn_id.into(); context.append_with_context_best_effort( codex_turn_id.clone(), RawTraceEventPayload::CodexTurnStarted { codex_turn_id, thread_id: context.thread_id.clone(), }, ); } /// Starts a first-class code-mode cell lifecycle and returns its trace handle. pub fn start_code_cell_trace( &self, codex_turn_id: impl Into, runtime_cell_id: impl Into, model_visible_call_id: impl Into, source_js: impl Into, ) -> CodeCellTraceContext { let context = self.code_cell_trace_context(codex_turn_id, runtime_cell_id); context.record_started(model_visible_call_id, source_js); context } /// Builds a trace handle for an already-started code-mode runtime cell. pub fn code_cell_trace_context( &self, codex_turn_id: impl Into, runtime_cell_id: impl Into, ) -> CodeCellTraceContext { let ThreadTraceContextState::Enabled(context) = &self.state else { return CodeCellTraceContext::disabled(); }; CodeCellTraceContext::enabled( Arc::clone(&context.writer), context.thread_id.clone(), codex_turn_id, runtime_cell_id, ) } /// Starts one dispatch-level tool lifecycle and returns its trace handle. /// /// `invocation` is lazy because adapting core tool objects into trace-owned /// payloads can clone large arguments. Disabled tracing should not pay that /// cost on the hot tool-dispatch path. pub fn start_tool_dispatch_trace( &self, invocation: impl FnOnce() -> Option, ) -> ToolDispatchTraceContext { let ThreadTraceContextState::Enabled(context) = &self.state else { return ToolDispatchTraceContext::disabled(); }; let Some(invocation) = invocation() else { return ToolDispatchTraceContext::disabled(); }; ToolDispatchTraceContext::start(Arc::clone(&context.writer), invocation) } /// Builds reusable inference trace context for one Codex turn. /// /// The returned context is intentionally not "an inference call" yet. /// Transport code owns retry/fallback attempts and calls `start_attempt` /// only after it has built the concrete request payload for that attempt. pub fn inference_trace_context( &self, codex_turn_id: impl Into, model: impl Into, provider_name: impl Into, ) -> InferenceTraceContext { let ThreadTraceContextState::Enabled(context) = &self.state else { return InferenceTraceContext::disabled(); }; InferenceTraceContext::enabled( Arc::clone(&context.writer), context.thread_id.clone(), codex_turn_id.into(), model.into(), provider_name.into(), ) } /// Builds remote-compaction trace context for one checkpoint. /// /// Rollout tracing currently has a first-class checkpoint model only for remote compaction. /// The compact endpoint is a model-facing request whose output replaces live history, so it /// needs both request/response attempt events and a later checkpoint event when processed /// replacement history is installed. pub fn compaction_trace_context( &self, codex_turn_id: impl Into, compaction_id: impl Into, model: impl Into, provider_name: impl Into, ) -> CompactionTraceContext { let ThreadTraceContextState::Enabled(context) = &self.state else { return CompactionTraceContext::disabled(); }; CompactionTraceContext::enabled( Arc::clone(&context.writer), context.thread_id.clone(), codex_turn_id.into(), compaction_id.into(), model.into(), provider_name.into(), ) } /// Starts bridge correlation for one concrete MCP backend request. /// /// Dispatch-level tool IDs remain compact and UI-friendly. This UUID is /// deliberately separate: it is only for cross-process log joins where a /// rollout-local counter would collide across samples. pub fn start_mcp_call_trace(&self, tool_call_id: impl Into) -> McpCallTraceContext { let ThreadTraceContextState::Enabled(context) = &self.state else { return McpCallTraceContext::disabled(); }; let mcp_call_id = Uuid::new_v4().to_string(); let trace = McpCallTraceContext::enabled(mcp_call_id.clone()); context.append_best_effort(RawTraceEventPayload::McpToolCallCorrelationAssigned { tool_call_id: tool_call_id.into(), mcp_call_id, }); trace } } fn start_root_in_root( root: &Path, metadata: ThreadStartedTraceMetadata, limits: TraceCaptureLimits, ) -> anyhow::Result { let trace_id = Uuid::new_v4().to_string(); let thread_id = metadata.thread_id.clone(); let bundle_dir = root.join(format!("trace-{trace_id}-{thread_id}")); let writer = TraceWriter::create_with_limits( &bundle_dir, trace_id.clone(), thread_id.clone(), thread_id.clone(), limits, )?; let writer = Arc::new(writer); writer.append(RawTraceEventPayload::RolloutStarted { trace_id, root_thread_id: thread_id.clone(), })?; debug!("recording rollout trace at {}", bundle_dir.display()); ThreadTraceContext::start(writer, thread_id, metadata) } impl EnabledThreadTraceContext { fn append_best_effort(&self, payload: RawTraceEventPayload) { if let Err(err) = self .writer .enqueue_event(RawTraceEventContext::default(), payload) { warn!("failed to append rollout trace event: {err:#}"); } } fn append_with_context_best_effort( &self, codex_turn_id: CodexTurnId, payload: RawTraceEventPayload, ) { let event_context = RawTraceEventContext { thread_id: Some(self.thread_id.clone()), codex_turn_id: Some(codex_turn_id), }; if let Err(err) = self.writer.enqueue_event(event_context, payload) { warn!("failed to append rollout trace event: {err:#}"); } } } #[cfg(test)] #[path = "thread_tests.rs"] mod tests;