mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Move rollout trace recording to a background worker (#61253)
## Why Recording inference starts synchronously serializes prompts and writes trace payloads before model work can proceed. Defer that work so inference dispatch can continue while the trace is recorded. ## What changed - Use one worker per trace bundle to write events in submission order, with up to eight pending jobs. - Share prepared prompts and encoded requests with the worker, and build remote compaction snapshots there. Write payloads before publishing events that reference them. - Add `TraceWriter::flush` and `ThreadTraceContext::flush` for live reads. Drain accepted jobs and seal the bundle on shutdown; suppress the closing marker after a recording failure. ## Testing Add tests for deferred prompt serialization, ordered start and completion events, shutdown draining, and serialization failures that preserve earlier evidence without publishing a closing marker. Verify compaction snapshots are built on the worker and skipped when tracing is disabled. GitOrigin-RevId: 32fd787bb1d281f1814df2f08c73b667628f8add
This commit is contained in:
committed by
copyberry
parent
aa5f492031
commit
b8dde1eb40
@@ -16,7 +16,6 @@ use crate::model::CodeCellRuntimeStatus;
|
||||
use crate::model::CodexTurnId;
|
||||
use crate::model::ModelVisibleCallId;
|
||||
use crate::payload::RawPayloadKind;
|
||||
use crate::payload::RawPayloadRef;
|
||||
use crate::raw_event::RawTraceEventContext;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
use crate::writer::TraceWriter;
|
||||
@@ -86,14 +85,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
if let Err(err) = context.writer.enqueue_event(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawTraceEventPayload::CodeCellStarted {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
model_visible_call_id: model_visible_call_id.into(),
|
||||
source_js: source_js.into(),
|
||||
},
|
||||
);
|
||||
) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Records the first response returned by the public code-mode `exec` tool.
|
||||
@@ -106,12 +107,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::CodeCellInitialResponse {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
status: code_cell_status_for_runtime_response(response),
|
||||
response_payload: code_cell_response_payload(context, response),
|
||||
let runtime_cell_id = context.runtime_cell_id.clone();
|
||||
let status = code_cell_status_for_runtime_response(response);
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
move |reference| RawTraceEventPayload::CodeCellInitialResponse {
|
||||
runtime_cell_id,
|
||||
status,
|
||||
response_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -121,12 +126,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::CodeCellEnded {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
status: code_cell_status_for_runtime_response(response),
|
||||
response_payload: code_cell_response_payload(context, response),
|
||||
let runtime_cell_id = context.runtime_cell_id.clone();
|
||||
let status = code_cell_status_for_runtime_response(response);
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
move |reference| RawTraceEventPayload::CodeCellEnded {
|
||||
runtime_cell_id,
|
||||
status,
|
||||
response_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -145,41 +154,3 @@ fn code_cell_status_for_runtime_response(response: &RuntimeResponse) -> CodeCell
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn code_cell_response_payload(
|
||||
context: &EnabledCodeCellTraceContext,
|
||||
response: &RuntimeResponse,
|
||||
) -> Option<RawPayloadRef> {
|
||||
write_json_payload_best_effort(
|
||||
&context.writer,
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
)
|
||||
}
|
||||
|
||||
fn write_json_payload_best_effort(
|
||||
writer: &TraceWriter,
|
||||
kind: RawPayloadKind,
|
||||
payload: &impl Serialize,
|
||||
) -> Option<RawPayloadRef> {
|
||||
match writer.write_json_payload(kind, payload) {
|
||||
Ok(payload_ref) => Some(payload_ref),
|
||||
Err(err) => {
|
||||
warn!("failed to write rollout trace payload: {err:#}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_with_context_best_effort(
|
||||
context: &EnabledCodeCellTraceContext,
|
||||
payload: RawTraceEventPayload,
|
||||
) {
|
||||
let event_context = RawTraceEventContext {
|
||||
thread_id: Some(context.thread_id.clone()),
|
||||
codex_turn_id: Some(context.codex_turn_id.clone()),
|
||||
};
|
||||
if let Err(err) = context.writer.append_with_context(event_context, payload) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user