Move rollout trace recording to a background worker (#61253)

## Why

Recording inference starts synchronously serializes prompts and writes trace payloads before model work can proceed. Defer that work so inference dispatch can continue while the trace is recorded.

## What changed

- Use one worker per trace bundle to write events in submission order, with up to eight pending jobs.
- Share prepared prompts and encoded requests with the worker, and build remote compaction snapshots there. Write payloads before publishing events that reference them.
- Add `TraceWriter::flush` and `ThreadTraceContext::flush` for live reads. Drain accepted jobs and seal the bundle on shutdown; suppress the closing marker after a recording failure.

## Testing

Add tests for deferred prompt serialization, ordered start and completion events, shutdown draining, and serialization failures that preserve earlier evidence without publishing a closing marker. Verify compaction snapshots are built on the worker and skipped when tracing is disabled.

GitOrigin-RevId: 32fd787bb1d281f1814df2f08c73b667628f8add
This commit is contained in:
cassirer-openai
2026-11-13 04:36:03 +00:00
committed by copyberry
parent aa5f492031
commit b8dde1eb40
26 changed files with 599 additions and 442 deletions
+3 -3
View File
@@ -167,7 +167,7 @@ async fn responses_stream_includes_subagent_header_on_review() {
let mut stream = client_session
.stream(
&prompt,
&Arc::new(prompt),
&model_info,
&session_telemetry,
effort,
@@ -317,7 +317,7 @@ async fn responses_stream_includes_subagent_header_on_other() {
let mut stream = client_session
.stream(
&prompt,
&Arc::new(prompt),
&model_info,
&session_telemetry,
effort,
@@ -452,7 +452,7 @@ async fn responses_respects_model_info_overrides_from_config() {
let mut stream = client_session
.stream(
&prompt,
&Arc::new(prompt),
&model_info,
&session_telemetry,
effort,
+2 -2
View File
@@ -1880,7 +1880,7 @@ async fn send_request_with_provider(provider: ModelProviderInfo) {
let mut stream = client_session
.stream(
&prompt,
&Arc::new(prompt),
&model_info,
&session_telemetry,
effort,
@@ -3660,7 +3660,7 @@ async fn azure_responses_request_does_not_store_and_preserves_prefixed_item_ids(
let mut stream = client_session
.stream(
&prompt,
&Arc::new(prompt),
&model_info,
&session_telemetry,
effort,
+14 -15
View File
@@ -962,7 +962,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() {
.expect("turn started");
let inference_trace = InferenceTraceContext::enabled(
writer,
Arc::clone(&writer),
harness.thread_id.to_string(),
"turn-1".to_string(),
harness.model_info.slug.clone(),
@@ -991,7 +991,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() {
}
let attribution = serde_json::json!([{"provenance": {"type": "user"}}]);
prompt.input.push(serde_json::from_value(serde_json::json!({
Arc::make_mut(&mut prompt).input.push(serde_json::from_value(serde_json::json!({
"type": "message", "role": "user", "content": [{"type": "input_text", "text": "follow up"}],
"internal_chat_message_metadata_passthrough": {"content_item_metadata": attribution}
})).unwrap());
@@ -1021,6 +1021,7 @@ async fn responses_websocket_request_prewarm_traces_logical_request() {
assert_eq!(follow_up["previous_response_id"].as_str(), Some("warm-1"));
assert_eq!(follow_up["input"], serde_json::json!([]));
writer.flush().expect("publish the live trace");
let rollout = replay_bundle(trace_dir.path()).expect("replay trace");
let mut reader = codex_rollout_trace::PayloadReader::new(trace_dir.path());
assert_eq!(rollout.inference_calls.len(), 2);
@@ -2555,7 +2556,7 @@ async fn responses_websocket_v2_creates_without_previous_response_id_when_non_in
let mut session = harness.client.new_session();
let prompt_one = prompt_with_input(vec![message_item("hello")]);
let mut prompt_two = prompt_with_input(vec![message_item("hello"), message_item("second")]);
prompt_two.output_schema = output_schema;
Arc::make_mut(&mut prompt_two).output_schema = output_schema;
let mut model_info = harness.model_info.clone();
if let Some(model) = model {
model_info.slug = model.to_string();
@@ -2834,20 +2835,18 @@ fn assistant_message_item(id: &str, text: &str) -> ResponseItem {
}
}
fn prompt_with_input(input: Vec<ResponseItem>) -> Prompt {
let mut prompt = Prompt::default();
prompt.input = input;
prompt.base_instructions.text.clear();
prompt
fn prompt_with_input(input: Vec<ResponseItem>) -> Arc<Prompt> {
prompt_with_input_and_instructions(input, "")
}
fn prompt_with_input_and_instructions(input: Vec<ResponseItem>, instructions: &str) -> Prompt {
let mut prompt = prompt_with_input(input);
fn prompt_with_input_and_instructions(input: Vec<ResponseItem>, instructions: &str) -> Arc<Prompt> {
let mut prompt = Prompt::default();
prompt.input = input;
prompt.base_instructions = BaseInstructions {
text: instructions.to_string(),
provenance: None,
};
prompt
Arc::new(prompt)
}
fn websocket_provider(server: &WebSocketTestServer) -> ModelProviderInfo {
@@ -3271,7 +3270,7 @@ fn assert_counter<const N: usize>(
async fn stream_until_complete(
client_session: &mut ModelClientSession,
harness: &WebsocketTestHarness,
prompt: &Prompt,
prompt: &Arc<Prompt>,
) {
stream_until_complete_with_service_tier(
client_session,
@@ -3285,7 +3284,7 @@ async fn stream_until_complete(
async fn stream_until_complete_with_model_info(
client_session: &mut ModelClientSession,
harness: &WebsocketTestHarness,
prompt: &Prompt,
prompt: &Arc<Prompt>,
model_info: &ModelInfo,
expected_response_id: &str,
) {
@@ -3318,7 +3317,7 @@ async fn stream_until_complete_with_model_info(
async fn stream_until_complete_with_service_tier(
client_session: &mut ModelClientSession,
harness: &WebsocketTestHarness,
prompt: &Prompt,
prompt: &Arc<Prompt>,
service_tier: Option<ServiceTier>,
) {
let responses_metadata = turn_metadata(harness, /*turn_id*/ None);
@@ -3335,7 +3334,7 @@ async fn stream_until_complete_with_service_tier(
async fn stream_until_complete_with_metadata(
client_session: &mut ModelClientSession,
harness: &WebsocketTestHarness,
prompt: &Prompt,
prompt: &Arc<Prompt>,
service_tier: Option<ServiceTier>,
responses_metadata: &CodexResponsesMetadata,
) {