Move rollout trace recording to a background worker (#61253)

## Why

Recording inference starts synchronously serializes prompts and writes trace payloads before model work can proceed. Defer that work so inference dispatch can continue while the trace is recorded.

## What changed

- Use one worker per trace bundle to write events in submission order, with up to eight pending jobs.
- Share prepared prompts and encoded requests with the worker, and build remote compaction snapshots there. Write payloads before publishing events that reference them.
- Add `TraceWriter::flush` and `ThreadTraceContext::flush` for live reads. Drain accepted jobs and seal the bundle on shutdown; suppress the closing marker after a recording failure.

## Testing

Add tests for deferred prompt serialization, ordered start and completion events, shutdown draining, and serialization failures that preserve earlier evidence without publishing a closing marker. Verify compaction snapshots are built on the worker and skipped when tracing is disabled.

GitOrigin-RevId: 32fd787bb1d281f1814df2f08c73b667628f8add
This commit is contained in:
cassirer-openai
2026-11-13 04:36:03 +00:00
committed by copyberry
parent aa5f492031
commit b8dde1eb40
26 changed files with 599 additions and 442 deletions
+16 -43
View File
@@ -14,7 +14,6 @@ use codex_protocol::models::SearchToolCallParams;
use serde::Serialize;
use serde_json::Value as JsonValue;
use serde_json::json;
use tracing::warn;
use crate::model::AgentThreadId;
use crate::model::CodeModeRuntimeToolId;
@@ -25,7 +24,6 @@ use crate::model::ToolCallId;
use crate::model::ToolCallKind;
use crate::model::ToolCallSummary;
use crate::payload::RawPayloadKind;
use crate::payload::RawPayloadRef;
use crate::raw_event::RawToolCallRequester;
use crate::raw_event::RawTraceEventContext;
use crate::raw_event::RawTraceEventPayload;
@@ -201,15 +199,16 @@ fn record_started(context: &EnabledToolDispatchTraceContext, invocation: ToolDis
tool_namespace: tool_namespace.as_deref(),
payload: &payload,
};
let request_payload =
write_json_payload_best_effort(&context.writer, RawPayloadKind::ToolInvocation, &request);
let (model_visible_call_id, code_mode_runtime_tool_id, requester) =
requester_fields(invocation.requester);
append_with_context_best_effort(
context,
RawTraceEventPayload::ToolCallStarted {
tool_call_id: context.tool_call_id.clone(),
let tool_call_id = context.tool_call_id.clone();
let _ = context.writer.event_with_payload(
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
RawPayloadKind::ToolInvocation,
&request,
move |reference| RawTraceEventPayload::ToolCallStarted {
tool_call_id,
model_visible_call_id,
code_mode_runtime_tool_id,
requester,
@@ -219,7 +218,7 @@ fn record_started(context: &EnabledToolDispatchTraceContext, invocation: ToolDis
input_preview,
output_preview: None,
},
invocation_payload: request_payload,
invocation_payload: Some(reference),
},
);
}
@@ -340,45 +339,19 @@ fn append_tool_call_ended(
status: ExecutionStatus,
response: &DispatchedToolTraceResponse<'_>,
) {
let response_payload =
write_json_payload_best_effort(&context.writer, RawPayloadKind::ToolResult, response);
append_with_context_best_effort(
context,
RawTraceEventPayload::ToolCallEnded {
tool_call_id: context.tool_call_id.clone(),
let tool_call_id = context.tool_call_id.clone();
let _ = context.writer.event_with_payload(
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
RawPayloadKind::ToolResult,
response,
move |reference| RawTraceEventPayload::ToolCallEnded {
tool_call_id,
status,
result_payload: response_payload,
result_payload: Some(reference),
},
);
}
fn write_json_payload_best_effort(
writer: &TraceWriter,
kind: RawPayloadKind,
payload: &impl Serialize,
) -> Option<RawPayloadRef> {
match writer.write_json_payload(kind, payload) {
Ok(payload_ref) => Some(payload_ref),
Err(err) => {
warn!("failed to write rollout trace payload: {err:#}");
None
}
}
}
fn append_with_context_best_effort(
context: &EnabledToolDispatchTraceContext,
payload: RawTraceEventPayload,
) {
let event_context = RawTraceEventContext {
thread_id: Some(context.thread_id.clone()),
codex_turn_id: Some(context.codex_turn_id.clone()),
};
if let Err(err) = context.writer.append_with_context(event_context, payload) {
warn!("failed to append rollout trace event: {err:#}");
}
}
#[cfg(test)]
mod tests {
use super::*;