Move rollout trace recording to a background worker (#61253)

## Why

Recording inference starts synchronously serializes prompts and writes trace payloads before model work can proceed. Defer that work so inference dispatch can continue while the trace is recorded.

## What changed

- Use one worker per trace bundle to write events in submission order, with up to eight pending jobs.
- Share prepared prompts and encoded requests with the worker, and build remote compaction snapshots there. Write payloads before publishing events that reference them.
- Add `TraceWriter::flush` and `ThreadTraceContext::flush` for live reads. Drain accepted jobs and seal the bundle on shutdown; suppress the closing marker after a recording failure.

## Testing

Add tests for deferred prompt serialization, ordered start and completion events, shutdown draining, and serialization failures that preserve earlier evidence without publishing a closing marker. Verify compaction snapshots are built on the worker and skipped when tracing is disabled.

GitOrigin-RevId: 32fd787bb1d281f1814df2f08c73b667628f8add
This commit is contained in:
cassirer-openai
2026-11-13 04:36:03 +00:00
committed by copyberry
parent aa5f492031
commit b8dde1eb40
26 changed files with 599 additions and 442 deletions
+17 -3
View File
@@ -56,6 +56,7 @@ fn create_in_root_writes_replayable_lifecycle_events() -> anyhow::Result<()> {
thread_trace.record_ended(RolloutStatus::Completed);
let bundle_dir = single_bundle_dir(temp.path())?;
thread_trace.flush()?;
let replayed = replay_bundle(&bundle_dir)?;
assert_eq!(replayed.status, RolloutStatus::Completed);
@@ -165,6 +166,7 @@ fn spawned_thread_start_appends_to_root_bundle() -> anyhow::Result<()> {
})?;
child_trace.record_ended(RolloutStatus::Completed);
let bundle_dir = single_bundle_dir(temp.path())?;
root_trace.flush()?;
let replayed = replay_bundle(&bundle_dir)?;
assert_eq!(fs::read_dir(temp.path())?.count(), 1);
@@ -262,7 +264,11 @@ fn disabled_thread_context_accepts_trace_calls_without_writing() -> anyhow::Resu
thread_trace.inference_trace_context("turn-1", "gpt-test", "test-provider");
let inference_attempt = inference_trace.start_attempt();
let input = serde_json::json!({ "kind": "inference" });
inference_attempt.record_started(serde_json::to_vec(&input)?, &input, || Ok(None));
inference_attempt.record_started(
serde_json::to_vec(&input)?,
std::sync::Arc::new(input),
|| Ok(None),
);
let token_usage: Option<codex_protocol::protocol::TokenUsage> = None;
inference_attempt.record_completed("response-1", Some("req-1"), &token_usage, &[]);
inference_attempt.record_failed("inference failed", /*upstream_request_id*/ None, &[]);
@@ -275,7 +281,7 @@ fn disabled_thread_context_accepts_trace_calls_without_writing() -> anyhow::Resu
);
assert!(!compaction_trace.is_enabled());
let compaction_attempt =
compaction_trace.start_attempt(&serde_json::json!({ "kind": "compaction" }));
compaction_trace.start_attempt::<()>(|| panic!("disabled tracing built the request"));
compaction_attempt.record_completed(&[]);
compaction_attempt.record_failed("compaction failed");
compaction_trace.record_installed(&CompactionCheckpointTracePayload {
@@ -308,11 +314,16 @@ fn compaction_contexts_share_identity_across_models() -> anyhow::Result<()> {
let compaction_trace =
thread_trace.compaction_trace_context("turn-1", "compaction-1", model, "test-provider");
assert!(compaction_trace.is_enabled());
let caller = std::thread::current().id();
compaction_trace
.start_attempt(&serde_json::json!({ "model": model }))
.start_attempt(move || {
assert_ne!(std::thread::current().id(), caller);
serde_json::json!({ "model": model })
})
.record_failed("test failure");
}
thread_trace.flush()?;
let replayed = replay_bundle(&single_bundle_dir(temp.path())?)?;
let mut attempts = replayed
.compaction_requests
@@ -340,6 +351,7 @@ fn protocol_wrapper_records_selected_events_as_raw_payloads() -> anyhow::Result<
thread_trace.record_protocol_event(&EventMsg::ShutdownComplete);
thread_trace.flush()?;
let event_log = fs::read_to_string(single_bundle_dir(temp.path())?.join("trace.jsonl"))?;
let protocol_event_seen = event_log.lines().any(|line| {
let event: crate::RawTraceEvent = serde_json::from_str(line).expect("raw trace event");
@@ -377,6 +389,7 @@ fn intercepted_exec_records_patch_without_a_terminal() -> anyhow::Result<()> {
},
}));
let bundle = single_bundle_dir(temp.path())?;
trace.flush()?;
let before = replay_bundle(&bundle)?.tool_calls["patch-call"].clone();
let changes = HashMap::from([(
PathBuf::from("hello.txt"),
@@ -415,6 +428,7 @@ fn intercepted_exec_records_patch_without_a_terminal() -> anyhow::Result<()> {
},
);
trace.flush()?;
let rollout = replay_bundle(&bundle)?;
let call = &rollout.tool_calls["patch-call"];
assert_eq!(call.kind, ToolCallKind::ApplyPatch);