mirror of
https://github.com/openai/codex.git
synced 2026-11-16 22:00:36 +00:00
Move rollout trace recording to a background worker (#61253)
## Why Recording inference starts synchronously serializes prompts and writes trace payloads before model work can proceed. Defer that work so inference dispatch can continue while the trace is recorded. ## What changed - Use one worker per trace bundle to write events in submission order, with up to eight pending jobs. - Share prepared prompts and encoded requests with the worker, and build remote compaction snapshots there. Write payloads before publishing events that reference them. - Add `TraceWriter::flush` and `ThreadTraceContext::flush` for live reads. Drain accepted jobs and seal the bundle on shutdown; suppress the closing marker after a recording failure. ## Testing Add tests for deferred prompt serialization, ordered start and completion events, shutdown draining, and serialization failures that preserve earlier evidence without publishing a closing marker. Verify compaction snapshots are built on the worker and skipped when tracing is disabled. GitOrigin-RevId: 32fd787bb1d281f1814df2f08c73b667628f8add
This commit is contained in:
committed by
copyberry
parent
aa5f492031
commit
b8dde1eb40
@@ -16,7 +16,6 @@ use crate::model::CodeCellRuntimeStatus;
|
||||
use crate::model::CodexTurnId;
|
||||
use crate::model::ModelVisibleCallId;
|
||||
use crate::payload::RawPayloadKind;
|
||||
use crate::payload::RawPayloadRef;
|
||||
use crate::raw_event::RawTraceEventContext;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
use crate::writer::TraceWriter;
|
||||
@@ -86,14 +85,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
if let Err(err) = context.writer.enqueue_event(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawTraceEventPayload::CodeCellStarted {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
model_visible_call_id: model_visible_call_id.into(),
|
||||
source_js: source_js.into(),
|
||||
},
|
||||
);
|
||||
) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Records the first response returned by the public code-mode `exec` tool.
|
||||
@@ -106,12 +107,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::CodeCellInitialResponse {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
status: code_cell_status_for_runtime_response(response),
|
||||
response_payload: code_cell_response_payload(context, response),
|
||||
let runtime_cell_id = context.runtime_cell_id.clone();
|
||||
let status = code_cell_status_for_runtime_response(response);
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
move |reference| RawTraceEventPayload::CodeCellInitialResponse {
|
||||
runtime_cell_id,
|
||||
status,
|
||||
response_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -121,12 +126,16 @@ impl CodeCellTraceContext {
|
||||
let CodeCellTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::CodeCellEnded {
|
||||
runtime_cell_id: context.runtime_cell_id.clone(),
|
||||
status: code_cell_status_for_runtime_response(response),
|
||||
response_payload: code_cell_response_payload(context, response),
|
||||
let runtime_cell_id = context.runtime_cell_id.clone();
|
||||
let status = code_cell_status_for_runtime_response(response);
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
move |reference| RawTraceEventPayload::CodeCellEnded {
|
||||
runtime_cell_id,
|
||||
status,
|
||||
response_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -145,41 +154,3 @@ fn code_cell_status_for_runtime_response(response: &RuntimeResponse) -> CodeCell
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn code_cell_response_payload(
|
||||
context: &EnabledCodeCellTraceContext,
|
||||
response: &RuntimeResponse,
|
||||
) -> Option<RawPayloadRef> {
|
||||
write_json_payload_best_effort(
|
||||
&context.writer,
|
||||
RawPayloadKind::ToolResult,
|
||||
&CodeCellResponseTracePayload { response },
|
||||
)
|
||||
}
|
||||
|
||||
fn write_json_payload_best_effort(
|
||||
writer: &TraceWriter,
|
||||
kind: RawPayloadKind,
|
||||
payload: &impl Serialize,
|
||||
) -> Option<RawPayloadRef> {
|
||||
match writer.write_json_payload(kind, payload) {
|
||||
Ok(payload_ref) => Some(payload_ref),
|
||||
Err(err) => {
|
||||
warn!("failed to write rollout trace payload: {err:#}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_with_context_best_effort(
|
||||
context: &EnabledCodeCellTraceContext,
|
||||
payload: RawTraceEventPayload,
|
||||
) {
|
||||
let event_context = RawTraceEventContext {
|
||||
thread_id: Some(context.thread_id.clone()),
|
||||
codex_turn_id: Some(context.codex_turn_id.clone()),
|
||||
};
|
||||
if let Err(err) = context.writer.append_with_context(event_context, payload) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ use std::sync::atomic::Ordering;
|
||||
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use serde::Serialize;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::inference::serialize_trace_items;
|
||||
use crate::model::AgentThreadId;
|
||||
@@ -120,19 +119,47 @@ impl CompactionTraceContext {
|
||||
matches!(self.state, CompactionTraceContextState::Enabled(_))
|
||||
}
|
||||
|
||||
/// Starts a new upstream attempt and records the exact compact endpoint request.
|
||||
pub fn start_attempt(&self, request: &impl Serialize) -> CompactionTraceAttempt {
|
||||
/// Starts an upstream attempt. The recording worker builds the snapshot if tracing is enabled.
|
||||
pub fn start_attempt<T: Serialize>(
|
||||
&self,
|
||||
request: impl FnOnce() -> T + Send + 'static,
|
||||
) -> CompactionTraceAttempt {
|
||||
let CompactionTraceContextState::Enabled(context) = &self.state else {
|
||||
return CompactionTraceAttempt::disabled();
|
||||
return CompactionTraceAttempt {
|
||||
state: CompactionTraceAttemptState::Disabled,
|
||||
};
|
||||
};
|
||||
|
||||
let compaction_request_id = next_compaction_request_id();
|
||||
let attempt = CompactionTraceAttempt {
|
||||
state: CompactionTraceAttemptState::Enabled(EnabledCompactionTraceAttempt {
|
||||
context: context.clone(),
|
||||
compaction_request_id: next_compaction_request_id(),
|
||||
compaction_request_id: compaction_request_id.clone(),
|
||||
}),
|
||||
};
|
||||
attempt.record_started(request);
|
||||
let compaction_id = context.compaction_id.clone();
|
||||
let thread_id = context.thread_id.clone();
|
||||
let codex_turn_id = context.codex_turn_id.clone();
|
||||
let model = context.model.clone();
|
||||
let provider_name = context.provider_name.clone();
|
||||
let _ = context.writer.defer_event(
|
||||
RawTraceEventContext::turn(&thread_id, &codex_turn_id),
|
||||
move |bundle| {
|
||||
let request_payload = bundle.write_body(
|
||||
RawPayloadKind::CompactionRequest,
|
||||
&serde_json::to_vec(&request())?,
|
||||
)?;
|
||||
Ok(RawTraceEventPayload::CompactionRequestStarted {
|
||||
compaction_id,
|
||||
compaction_request_id,
|
||||
thread_id,
|
||||
codex_turn_id,
|
||||
model,
|
||||
provider_name,
|
||||
request_payload,
|
||||
})
|
||||
},
|
||||
);
|
||||
attempt
|
||||
}
|
||||
|
||||
@@ -144,67 +171,20 @@ impl CompactionTraceContext {
|
||||
let CompactionTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
let checkpoint_payload = match context
|
||||
.writer
|
||||
.write_json_payload(RawPayloadKind::CompactionCheckpoint, checkpoint)
|
||||
{
|
||||
Ok(payload_ref) => payload_ref,
|
||||
Err(err) => {
|
||||
warn!("failed to write rollout trace payload: {err:#}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let event_context = RawTraceEventContext {
|
||||
thread_id: Some(context.thread_id.clone()),
|
||||
codex_turn_id: Some(context.codex_turn_id.clone()),
|
||||
};
|
||||
if let Err(err) = context.writer.append_with_context(
|
||||
event_context,
|
||||
RawTraceEventPayload::CompactionInstalled {
|
||||
compaction_id: context.compaction_id.clone(),
|
||||
let compaction_id = context.compaction_id.clone();
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::CompactionCheckpoint,
|
||||
checkpoint,
|
||||
move |checkpoint_payload| RawTraceEventPayload::CompactionInstalled {
|
||||
compaction_id,
|
||||
checkpoint_payload,
|
||||
},
|
||||
) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl CompactionTraceAttempt {
|
||||
/// Builds an attempt that records nothing.
|
||||
fn disabled() -> Self {
|
||||
Self {
|
||||
state: CompactionTraceAttemptState::Disabled,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_started(&self, request: &impl Serialize) {
|
||||
let CompactionTraceAttemptState::Enabled(attempt) = &self.state else {
|
||||
return;
|
||||
};
|
||||
let Some(request_payload) = write_json_payload_best_effort(
|
||||
&attempt.context.writer,
|
||||
RawPayloadKind::CompactionRequest,
|
||||
request,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
|
||||
append_with_context_best_effort(
|
||||
&attempt.context,
|
||||
RawTraceEventPayload::CompactionRequestStarted {
|
||||
compaction_id: attempt.context.compaction_id.clone(),
|
||||
compaction_request_id: attempt.compaction_request_id.clone(),
|
||||
thread_id: attempt.context.thread_id.clone(),
|
||||
codex_turn_id: attempt.context.codex_turn_id.clone(),
|
||||
model: attempt.context.model.clone(),
|
||||
provider_name: attempt.context.provider_name.clone(),
|
||||
request_payload,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Records the non-streaming compact endpoint response payload.
|
||||
///
|
||||
/// Compaction responses use the same response-item preservation rules as
|
||||
@@ -215,19 +195,16 @@ impl CompactionTraceAttempt {
|
||||
return;
|
||||
};
|
||||
let response_payload = TracedCompactionCompleted { output_items };
|
||||
let Some(response_payload) = write_json_payload_best_effort(
|
||||
&attempt.context.writer,
|
||||
let ctx = &attempt.context;
|
||||
let compaction_id = ctx.compaction_id.clone();
|
||||
let compaction_request_id = attempt.compaction_request_id.clone();
|
||||
let _ = ctx.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&ctx.thread_id, &ctx.codex_turn_id),
|
||||
RawPayloadKind::CompactionResponse,
|
||||
&response_payload,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
|
||||
append_with_context_best_effort(
|
||||
&attempt.context,
|
||||
RawTraceEventPayload::CompactionRequestCompleted {
|
||||
compaction_id: attempt.context.compaction_id.clone(),
|
||||
compaction_request_id: attempt.compaction_request_id.clone(),
|
||||
move |response_payload| RawTraceEventPayload::CompactionRequestCompleted {
|
||||
compaction_id,
|
||||
compaction_request_id,
|
||||
response_payload,
|
||||
},
|
||||
);
|
||||
@@ -246,8 +223,8 @@ impl CompactionTraceAttempt {
|
||||
let CompactionTraceAttemptState::Enabled(attempt) = &self.state else {
|
||||
return;
|
||||
};
|
||||
append_with_context_best_effort(
|
||||
&attempt.context,
|
||||
let _ = attempt.context.writer.enqueue_event(
|
||||
RawTraceEventContext::turn(&attempt.context.thread_id, &attempt.context.codex_turn_id),
|
||||
RawTraceEventPayload::CompactionRequestFailed {
|
||||
compaction_id: attempt.context.compaction_id.clone(),
|
||||
compaction_request_id: attempt.compaction_request_id.clone(),
|
||||
@@ -261,22 +238,3 @@ fn next_compaction_request_id() -> CompactionRequestId {
|
||||
let ordinal = NEXT_COMPACTION_REQUEST.fetch_add(1, Ordering::Relaxed);
|
||||
format!("compaction_request:{ordinal}")
|
||||
}
|
||||
|
||||
fn write_json_payload_best_effort(
|
||||
writer: &TraceWriter,
|
||||
kind: RawPayloadKind,
|
||||
payload: &impl Serialize,
|
||||
) -> Option<crate::RawPayloadRef> {
|
||||
writer.write_json_payload(kind, payload).ok()
|
||||
}
|
||||
|
||||
fn append_with_context_best_effort(
|
||||
context: &EnabledCompactionTraceContext,
|
||||
payload: RawTraceEventPayload,
|
||||
) {
|
||||
let event_context = RawTraceEventContext {
|
||||
thread_id: Some(context.thread_id.clone()),
|
||||
codex_turn_id: Some(context.codex_turn_id.clone()),
|
||||
};
|
||||
let _ = context.writer.append_with_context(event_context, payload);
|
||||
}
|
||||
|
||||
@@ -177,11 +177,11 @@ impl InferenceTraceAttempt {
|
||||
|
||||
/// Captures the transport's encoded JSON and the immutable Core prompt that produced it.
|
||||
/// A continuation from an unrecorded warmup also supplies its full formatted context.
|
||||
pub fn record_started(
|
||||
pub fn record_started<P: Serialize + Send + Sync + 'static>(
|
||||
&self,
|
||||
outgoing: impl AsRef<[u8]>,
|
||||
prepared_prompt: &impl Serialize,
|
||||
warmup_context: impl FnOnce() -> serde_json::Result<Option<Vec<u8>>>,
|
||||
outgoing: impl AsRef<[u8]> + Send + 'static,
|
||||
prepared_prompt: Arc<P>,
|
||||
warmup_context: impl FnOnce() -> serde_json::Result<Option<Vec<u8>>> + Send + 'static,
|
||||
) {
|
||||
let InferenceTraceAttemptState::Enabled(attempt) = &self.state else {
|
||||
return;
|
||||
@@ -192,14 +192,14 @@ impl InferenceTraceAttempt {
|
||||
let inference_call_id = attempt.inference_call_id.clone();
|
||||
let model = ctx.model.clone();
|
||||
let provider_name = ctx.provider_name.clone();
|
||||
let start = ctx.writer.record_event(
|
||||
let start = ctx.writer.defer_event(
|
||||
RawTraceEventContext::turn(&thread_id, &codex_turn_id),
|
||||
move |bundle| {
|
||||
let request_payload =
|
||||
bundle.write_body(RawPayloadKind::InferenceRequest, outgoing.as_ref())?;
|
||||
let prepared_prompt_payload = bundle.write_body(
|
||||
RawPayloadKind::PreparedPrompt,
|
||||
&serde_json::to_vec(prepared_prompt)?,
|
||||
&serde_json::to_vec(prepared_prompt.as_ref())?,
|
||||
)?;
|
||||
let warmup_context_payload = warmup_context()?
|
||||
.map(|bytes| bundle.write_body(RawPayloadKind::WarmupContext, &bytes))
|
||||
@@ -364,7 +364,7 @@ fn record_partial_response(
|
||||
let ctx = &attempt.context;
|
||||
let envelope = RawTraceEventContext::turn(&ctx.thread_id, &ctx.codex_turn_id);
|
||||
if output_items.is_empty() {
|
||||
let _ = ctx.writer.append_with_context(envelope, event(None));
|
||||
let _ = ctx.writer.enqueue_event(envelope, event(None));
|
||||
} else {
|
||||
let payload = TracedResponseStreamOutput {
|
||||
response_id: None,
|
||||
@@ -469,9 +469,12 @@ mod tests {
|
||||
"content": [{"type": "input_text", "text": "hello"}]
|
||||
}],
|
||||
});
|
||||
attempt.record_started(serde_json::to_vec(&request)?, &request, || Ok(None));
|
||||
attempt.record_started(serde_json::to_vec(&request)?, Arc::new(request), || {
|
||||
Ok(None)
|
||||
});
|
||||
attempt.record_completed("resp-1", Some("req-1"), &None, &[]);
|
||||
|
||||
writer.flush()?;
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let inference = rollout
|
||||
.inference_calls
|
||||
@@ -521,3 +524,7 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "inference_recording_tests.rs"]
|
||||
mod recording_tests;
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde::Serialize;
|
||||
use serde::Serializer;
|
||||
use tempfile::TempDir;
|
||||
|
||||
use crate::InferenceTraceAttempt;
|
||||
use crate::InferenceTraceContext;
|
||||
use crate::PayloadReader;
|
||||
use crate::RawPayloadKind;
|
||||
use crate::RawTraceEvent;
|
||||
use crate::RawTraceEventContext;
|
||||
use crate::RawTraceEventPayload;
|
||||
use crate::TraceWriter;
|
||||
|
||||
fn recording(dir: &TempDir) -> anyhow::Result<(Arc<TraceWriter>, InferenceTraceAttempt)> {
|
||||
let writer = Arc::new(TraceWriter::create(
|
||||
dir.path(),
|
||||
"trace".into(),
|
||||
"rollout".into(),
|
||||
"root".into(),
|
||||
)?);
|
||||
let attempt = InferenceTraceContext::enabled(
|
||||
Arc::clone(&writer),
|
||||
"root".into(),
|
||||
"turn".into(),
|
||||
"model".into(),
|
||||
"provider".into(),
|
||||
)
|
||||
.start_attempt();
|
||||
Ok((writer, attempt))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inference_dispatch_can_finish_before_prepared_prompt_serialization() -> anyhow::Result<()> {
|
||||
struct Prompt {
|
||||
caller: thread::ThreadId,
|
||||
release: Mutex<mpsc::Receiver<()>>,
|
||||
}
|
||||
impl Serialize for Prompt {
|
||||
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
assert_ne!(thread::current().id(), self.caller);
|
||||
self.release.lock().unwrap().recv().unwrap();
|
||||
serializer.serialize_str("prepared")
|
||||
}
|
||||
}
|
||||
|
||||
let dir = TempDir::new()?;
|
||||
let (writer, attempt) = recording(&dir)?;
|
||||
let (release, gate) = mpsc::channel();
|
||||
let wire = br#"{ "input": [] }"#;
|
||||
attempt.record_started(
|
||||
wire,
|
||||
Arc::new(Prompt {
|
||||
caller: thread::current().id(),
|
||||
release: Mutex::new(gate),
|
||||
}),
|
||||
|| Ok(None),
|
||||
);
|
||||
attempt.record_completed("response", /*upstream_request_id*/ None, &None, &[]);
|
||||
release.send(())?;
|
||||
drop(attempt);
|
||||
drop(writer);
|
||||
let events = std::fs::read_to_string(dir.path().join("trace.jsonl"))?
|
||||
.lines()
|
||||
.map(serde_json::from_str)
|
||||
.collect::<Result<Vec<RawTraceEvent>, _>>()?;
|
||||
let [start, end, closed] = &events[..] else {
|
||||
panic!("shutdown must publish both accepted events before recording closure")
|
||||
};
|
||||
assert!(matches!(
|
||||
closed.payload,
|
||||
RawTraceEventPayload::RecordingCompleted
|
||||
));
|
||||
let RawTraceEventPayload::InferenceStarted {
|
||||
request_payload,
|
||||
prepared_prompt_payload,
|
||||
..
|
||||
} = &start.payload
|
||||
else {
|
||||
panic!("start must precede completion")
|
||||
};
|
||||
assert!(matches!(
|
||||
end.payload,
|
||||
RawTraceEventPayload::InferenceCompleted { .. }
|
||||
));
|
||||
let mut reader = PayloadReader::new(dir.path());
|
||||
assert_eq!(reader.read(request_payload)?, wire);
|
||||
assert_eq!(reader.read(prepared_prompt_payload)?, br#""prepared""#);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_serialization_failure_suppresses_the_start_and_closing_marker() -> anyhow::Result<()> {
|
||||
let dir = TempDir::new()?;
|
||||
let (writer, attempt) = recording(&dir)?;
|
||||
// JSON cannot encode an array as an object key.
|
||||
attempt.record_started(b"{}", Arc::new(BTreeMap::from([([1, 2], true)])), || {
|
||||
Ok(None)
|
||||
});
|
||||
assert!(writer.flush().is_err());
|
||||
drop(attempt);
|
||||
drop(writer);
|
||||
assert_eq!(std::fs::read_to_string(dir.path().join("trace.jsonl"))?, "");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn borrowed_response_encoding_failure_preserves_the_queued_start() -> anyhow::Result<()> {
|
||||
let dir = TempDir::new()?;
|
||||
let (writer, attempt) = recording(&dir)?;
|
||||
attempt.record_started(b"{}", Arc::new("prepared"), || Ok(None));
|
||||
// Handoff succeeds even though JSON cannot represent this response's map key.
|
||||
writer.event_with_payload(
|
||||
RawTraceEventContext::turn("root", "turn"),
|
||||
RawPayloadKind::InferenceResponse,
|
||||
&BTreeMap::from([([1, 2], true)]),
|
||||
|_| unreachable!("a failed payload has no reference to publish"),
|
||||
)?;
|
||||
assert!(
|
||||
writer
|
||||
.flush()
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("key must be a string")
|
||||
);
|
||||
drop(attempt);
|
||||
drop(writer);
|
||||
let log = std::fs::read_to_string(dir.path().join("trace.jsonl"))?;
|
||||
// Exactly one event survived. Parsing the whole log rejects a spurious closing marker.
|
||||
let event: RawTraceEvent = serde_json::from_str(&log)?;
|
||||
assert!(matches!(
|
||||
event.payload,
|
||||
RawTraceEventPayload::InferenceStarted { .. }
|
||||
));
|
||||
Ok(())
|
||||
}
|
||||
@@ -11,7 +11,6 @@
|
||||
//! variants a compile-time prompt to decide whether the trace should capture
|
||||
//! them.
|
||||
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::ExecCommandBeginEvent;
|
||||
use codex_protocol::protocol::ExecCommandEndEvent;
|
||||
@@ -42,7 +41,6 @@ pub(crate) fn codex_turn_trace_event(
|
||||
thread_id: AgentThreadId,
|
||||
default_turn_id: &str,
|
||||
event: &EventMsg,
|
||||
store_agent_message: impl FnOnce(&ResponseItem) -> Option<crate::RawPayloadRef>,
|
||||
) -> Option<CodexTurnTraceEvent> {
|
||||
match event {
|
||||
EventMsg::TurnStarted(event) => {
|
||||
@@ -82,16 +80,6 @@ pub(crate) fn codex_turn_trace_event(
|
||||
},
|
||||
})
|
||||
}
|
||||
EventMsg::RawResponseItem(event) => Some(CodexTurnTraceEvent {
|
||||
context_turn_id: default_turn_id.to_string(),
|
||||
payload: RawTraceEventPayload::HistoryItemAccepted {
|
||||
item_id: event.item.id()?.to_string(),
|
||||
agent_message_payload: match &event.item {
|
||||
item @ ResponseItem::AgentMessage { .. } => Some(store_agent_message(item)?),
|
||||
_ => None,
|
||||
},
|
||||
},
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +72,7 @@ fn mcp_calls_keep_callable_identity_and_classify_the_backend() -> anyhow::Result
|
||||
},
|
||||
},
|
||||
);
|
||||
writer.flush()?;
|
||||
let before = replay_bundle(temp.path())?;
|
||||
// Namespaced write_stdin calls must neither require terminal arguments
|
||||
// nor create terminal sessions when those arguments happen to be valid.
|
||||
|
||||
@@ -538,6 +538,7 @@ fn invalid_write_stdin_arguments_remain_failed_tool_calls() -> anyhow::Result<()
|
||||
);
|
||||
dispatch.record_failed("invalid arguments");
|
||||
|
||||
writer.flush()?;
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let call = &rollout.tool_calls["call-stdin"];
|
||||
assert_eq!(call.execution.status, ExecutionStatus::Failed);
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
//! through session code.
|
||||
|
||||
use codex_protocol::mcp::CallToolResult;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use codex_protocol::protocol::AgentStatus;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
@@ -26,7 +27,6 @@ use crate::ExecutionStatus;
|
||||
use crate::InferenceTraceContext;
|
||||
use crate::McpCallTraceContext;
|
||||
use crate::RawPayloadKind;
|
||||
use crate::RawPayloadRef;
|
||||
use crate::RawTraceEventContext;
|
||||
use crate::RawTraceEventPayload;
|
||||
use crate::RolloutStatus;
|
||||
@@ -130,13 +130,28 @@ impl ThreadTraceContext {
|
||||
root_thread_id,
|
||||
thread_id: metadata.thread_id.clone(),
|
||||
};
|
||||
record_thread_started(&context, metadata);
|
||||
context.writer.check()?;
|
||||
// Thread creation reports storage failures before returning the new handle.
|
||||
let metadata_payload = context
|
||||
.writer
|
||||
.write_json_payload(RawPayloadKind::SessionMetadata, &metadata)?;
|
||||
context.writer.append(RawTraceEventPayload::ThreadStarted {
|
||||
thread_id: metadata.thread_id,
|
||||
agent_path: metadata.agent_path,
|
||||
metadata_payload: Some(metadata_payload),
|
||||
})?;
|
||||
Ok(Self {
|
||||
state: ThreadTraceContextState::Enabled(context),
|
||||
})
|
||||
}
|
||||
|
||||
/// Publishes queued evidence before reading a snapshot of a running bundle.
|
||||
pub fn flush(&self) -> anyhow::Result<()> {
|
||||
match &self.state {
|
||||
ThreadTraceContextState::Disabled => Ok(()),
|
||||
ThreadTraceContextState::Enabled(context) => context.writer.flush(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this handle will write trace events.
|
||||
///
|
||||
/// Most methods have their own disabled fast path. Callers should branch on
|
||||
@@ -209,15 +224,16 @@ impl ThreadTraceContext {
|
||||
let Some(event_type) = wrapped_protocol_event_type(event) else {
|
||||
return;
|
||||
};
|
||||
let Some(event_payload) =
|
||||
context.write_json_payload_best_effort(RawPayloadKind::ProtocolEvent, event)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
context.append_best_effort(RawTraceEventPayload::ProtocolEventObserved {
|
||||
event_type: event_type.to_string(),
|
||||
event_payload,
|
||||
});
|
||||
let event_type = event_type.to_owned();
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::default(),
|
||||
RawPayloadKind::ProtocolEvent,
|
||||
event,
|
||||
move |event_payload| RawTraceEventPayload::ProtocolEventObserved {
|
||||
event_type,
|
||||
event_payload,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// Records turn lifecycle and history acceptance from existing protocol events.
|
||||
@@ -225,10 +241,30 @@ impl ThreadTraceContext {
|
||||
let ThreadTraceContextState::Enabled(context) = &self.state else {
|
||||
return;
|
||||
};
|
||||
if let EventMsg::RawResponseItem(event) = event {
|
||||
let Some(item_id) = event.item.id().map(ToString::to_string) else {
|
||||
return;
|
||||
};
|
||||
let accepted = move |agent_message_payload| RawTraceEventPayload::HistoryItemAccepted {
|
||||
item_id,
|
||||
agent_message_payload,
|
||||
};
|
||||
let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id);
|
||||
if matches!(event.item, ResponseItem::AgentMessage { .. }) {
|
||||
// Publish the receipt and its history event together, in worker order.
|
||||
let _ = context.writer.event_with_payload(
|
||||
turn,
|
||||
RawPayloadKind::ProtocolEvent,
|
||||
&event.item,
|
||||
move |payload| accepted(Some(payload)),
|
||||
);
|
||||
} else {
|
||||
let _ = context.writer.enqueue_event(turn, accepted(None));
|
||||
}
|
||||
return;
|
||||
}
|
||||
let Some(trace_event) =
|
||||
codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event, |item| {
|
||||
context.write_json_payload_best_effort(RawPayloadKind::ProtocolEvent, item)
|
||||
})
|
||||
codex_turn_trace_event(context.thread_id.clone(), default_turn_id, event)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
@@ -250,10 +286,43 @@ impl ThreadTraceContext {
|
||||
let Some(trace_event) = tool_runtime_trace_event(event) else {
|
||||
return;
|
||||
};
|
||||
let Some(payload) = context.raw_tool_runtime_payload(trace_event) else {
|
||||
return;
|
||||
};
|
||||
context.append_with_context_best_effort(codex_turn_id.into(), payload);
|
||||
let envelope = RawTraceEventContext::turn(&context.thread_id, &codex_turn_id.into());
|
||||
match trace_event {
|
||||
crate::protocol_event::ToolRuntimeTraceEvent::Started {
|
||||
tool_call_id,
|
||||
kind,
|
||||
payload,
|
||||
} => {
|
||||
let tool_call_id = tool_call_id.to_owned();
|
||||
let _ = context.writer.event_with_payload(
|
||||
envelope,
|
||||
RawPayloadKind::ToolRuntimeEvent,
|
||||
&payload,
|
||||
move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeStarted {
|
||||
tool_call_id,
|
||||
kind,
|
||||
runtime_payload,
|
||||
},
|
||||
);
|
||||
}
|
||||
crate::protocol_event::ToolRuntimeTraceEvent::Ended {
|
||||
tool_call_id,
|
||||
status,
|
||||
payload,
|
||||
} => {
|
||||
let tool_call_id = tool_call_id.to_owned();
|
||||
let _ = context.writer.event_with_payload(
|
||||
envelope,
|
||||
RawPayloadKind::ToolRuntimeEvent,
|
||||
&payload,
|
||||
move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeEnded {
|
||||
tool_call_id,
|
||||
status,
|
||||
runtime_payload,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Records the full MCP backend result before callbacks, sanitization, or UI truncation.
|
||||
@@ -271,20 +340,19 @@ impl ThreadTraceContext {
|
||||
struct Payload<'a> {
|
||||
result: &'a Result<CallToolResult, String>,
|
||||
}
|
||||
let Some(runtime_payload) = context
|
||||
.write_json_payload_best_effort(RawPayloadKind::ToolRuntimeEvent, &Payload { result })
|
||||
else {
|
||||
return;
|
||||
let tool_call_id = tool_call_id.to_owned();
|
||||
let status = if result.is_ok() {
|
||||
ExecutionStatus::Completed
|
||||
} else {
|
||||
ExecutionStatus::Failed
|
||||
};
|
||||
context.append_with_context_best_effort(
|
||||
codex_turn_id.to_owned(),
|
||||
RawTraceEventPayload::ToolCallRuntimeEnded {
|
||||
tool_call_id: tool_call_id.to_owned(),
|
||||
status: if result.is_ok() {
|
||||
ExecutionStatus::Completed
|
||||
} else {
|
||||
ExecutionStatus::Failed
|
||||
},
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, codex_turn_id),
|
||||
RawPayloadKind::ToolRuntimeEvent,
|
||||
&Payload { result },
|
||||
move |runtime_payload| RawTraceEventPayload::ToolCallRuntimeEnded {
|
||||
tool_call_id,
|
||||
status,
|
||||
runtime_payload,
|
||||
},
|
||||
);
|
||||
@@ -326,20 +394,21 @@ impl ThreadTraceContext {
|
||||
};
|
||||
let child_codex_turn_id = child_codex_turn_id.into();
|
||||
let parent_thread_id = parent_thread_id.into();
|
||||
let carried_payload =
|
||||
context.write_json_payload_best_effort(RawPayloadKind::AgentResult, payload);
|
||||
context.append_with_context_best_effort(
|
||||
child_codex_turn_id.clone(),
|
||||
RawTraceEventPayload::AgentResultObserved {
|
||||
edge_id: format!(
|
||||
"edge:agent_result:{}:{child_codex_turn_id}:{parent_thread_id}",
|
||||
context.thread_id
|
||||
),
|
||||
child_thread_id: context.thread_id.clone(),
|
||||
let child_thread_id = context.thread_id.clone();
|
||||
let message = payload.message.to_owned();
|
||||
let edge_id =
|
||||
format!("edge:agent_result:{child_thread_id}:{child_codex_turn_id}:{parent_thread_id}");
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&child_thread_id, &child_codex_turn_id),
|
||||
RawPayloadKind::AgentResult,
|
||||
payload,
|
||||
move |reference| RawTraceEventPayload::AgentResultObserved {
|
||||
edge_id,
|
||||
child_thread_id,
|
||||
child_codex_turn_id,
|
||||
parent_thread_id,
|
||||
message: payload.message.to_string(),
|
||||
carried_payload,
|
||||
message,
|
||||
carried_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -503,70 +572,12 @@ fn start_root_in_root(
|
||||
ThreadTraceContext::start(writer, thread_id, metadata)
|
||||
}
|
||||
|
||||
fn record_thread_started(
|
||||
context: &EnabledThreadTraceContext,
|
||||
metadata: ThreadStartedTraceMetadata,
|
||||
) {
|
||||
let metadata_payload =
|
||||
context.write_json_payload_best_effort(RawPayloadKind::SessionMetadata, &metadata);
|
||||
context.append_best_effort(RawTraceEventPayload::ThreadStarted {
|
||||
thread_id: metadata.thread_id,
|
||||
agent_path: metadata.agent_path,
|
||||
metadata_payload,
|
||||
});
|
||||
}
|
||||
|
||||
impl EnabledThreadTraceContext {
|
||||
fn write_json_payload_best_effort(
|
||||
&self,
|
||||
kind: RawPayloadKind,
|
||||
payload: &impl Serialize,
|
||||
) -> Option<RawPayloadRef> {
|
||||
match self.writer.write_json_payload(kind, payload) {
|
||||
Ok(payload_ref) => Some(payload_ref),
|
||||
Err(err) => {
|
||||
warn!("failed to write rollout trace payload: {err:#}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_tool_runtime_payload(
|
||||
&self,
|
||||
trace_event: crate::protocol_event::ToolRuntimeTraceEvent<'_>,
|
||||
) -> Option<RawTraceEventPayload> {
|
||||
match trace_event {
|
||||
crate::protocol_event::ToolRuntimeTraceEvent::Started {
|
||||
tool_call_id,
|
||||
kind,
|
||||
payload,
|
||||
} => {
|
||||
let runtime_payload = self
|
||||
.write_json_payload_best_effort(RawPayloadKind::ToolRuntimeEvent, &payload)?;
|
||||
Some(RawTraceEventPayload::ToolCallRuntimeStarted {
|
||||
tool_call_id: tool_call_id.to_string(),
|
||||
kind,
|
||||
runtime_payload,
|
||||
})
|
||||
}
|
||||
crate::protocol_event::ToolRuntimeTraceEvent::Ended {
|
||||
tool_call_id,
|
||||
status,
|
||||
payload,
|
||||
} => {
|
||||
let runtime_payload = self
|
||||
.write_json_payload_best_effort(RawPayloadKind::ToolRuntimeEvent, &payload)?;
|
||||
Some(RawTraceEventPayload::ToolCallRuntimeEnded {
|
||||
tool_call_id: tool_call_id.to_string(),
|
||||
status,
|
||||
runtime_payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_best_effort(&self, payload: RawTraceEventPayload) {
|
||||
if let Err(err) = self.writer.append(payload) {
|
||||
if let Err(err) = self
|
||||
.writer
|
||||
.enqueue_event(RawTraceEventContext::default(), payload)
|
||||
{
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
@@ -580,7 +591,7 @@ impl EnabledThreadTraceContext {
|
||||
thread_id: Some(self.thread_id.clone()),
|
||||
codex_turn_id: Some(codex_turn_id),
|
||||
};
|
||||
if let Err(err) = self.writer.append_with_context(event_context, payload) {
|
||||
if let Err(err) = self.writer.enqueue_event(event_context, payload) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ fn create_in_root_writes_replayable_lifecycle_events() -> anyhow::Result<()> {
|
||||
thread_trace.record_ended(RolloutStatus::Completed);
|
||||
|
||||
let bundle_dir = single_bundle_dir(temp.path())?;
|
||||
thread_trace.flush()?;
|
||||
let replayed = replay_bundle(&bundle_dir)?;
|
||||
|
||||
assert_eq!(replayed.status, RolloutStatus::Completed);
|
||||
@@ -165,6 +166,7 @@ fn spawned_thread_start_appends_to_root_bundle() -> anyhow::Result<()> {
|
||||
})?;
|
||||
child_trace.record_ended(RolloutStatus::Completed);
|
||||
let bundle_dir = single_bundle_dir(temp.path())?;
|
||||
root_trace.flush()?;
|
||||
let replayed = replay_bundle(&bundle_dir)?;
|
||||
|
||||
assert_eq!(fs::read_dir(temp.path())?.count(), 1);
|
||||
@@ -262,7 +264,11 @@ fn disabled_thread_context_accepts_trace_calls_without_writing() -> anyhow::Resu
|
||||
thread_trace.inference_trace_context("turn-1", "gpt-test", "test-provider");
|
||||
let inference_attempt = inference_trace.start_attempt();
|
||||
let input = serde_json::json!({ "kind": "inference" });
|
||||
inference_attempt.record_started(serde_json::to_vec(&input)?, &input, || Ok(None));
|
||||
inference_attempt.record_started(
|
||||
serde_json::to_vec(&input)?,
|
||||
std::sync::Arc::new(input),
|
||||
|| Ok(None),
|
||||
);
|
||||
let token_usage: Option<codex_protocol::protocol::TokenUsage> = None;
|
||||
inference_attempt.record_completed("response-1", Some("req-1"), &token_usage, &[]);
|
||||
inference_attempt.record_failed("inference failed", /*upstream_request_id*/ None, &[]);
|
||||
@@ -275,7 +281,7 @@ fn disabled_thread_context_accepts_trace_calls_without_writing() -> anyhow::Resu
|
||||
);
|
||||
assert!(!compaction_trace.is_enabled());
|
||||
let compaction_attempt =
|
||||
compaction_trace.start_attempt(&serde_json::json!({ "kind": "compaction" }));
|
||||
compaction_trace.start_attempt::<()>(|| panic!("disabled tracing built the request"));
|
||||
compaction_attempt.record_completed(&[]);
|
||||
compaction_attempt.record_failed("compaction failed");
|
||||
compaction_trace.record_installed(&CompactionCheckpointTracePayload {
|
||||
@@ -308,11 +314,16 @@ fn compaction_contexts_share_identity_across_models() -> anyhow::Result<()> {
|
||||
let compaction_trace =
|
||||
thread_trace.compaction_trace_context("turn-1", "compaction-1", model, "test-provider");
|
||||
assert!(compaction_trace.is_enabled());
|
||||
let caller = std::thread::current().id();
|
||||
compaction_trace
|
||||
.start_attempt(&serde_json::json!({ "model": model }))
|
||||
.start_attempt(move || {
|
||||
assert_ne!(std::thread::current().id(), caller);
|
||||
serde_json::json!({ "model": model })
|
||||
})
|
||||
.record_failed("test failure");
|
||||
}
|
||||
|
||||
thread_trace.flush()?;
|
||||
let replayed = replay_bundle(&single_bundle_dir(temp.path())?)?;
|
||||
let mut attempts = replayed
|
||||
.compaction_requests
|
||||
@@ -340,6 +351,7 @@ fn protocol_wrapper_records_selected_events_as_raw_payloads() -> anyhow::Result<
|
||||
|
||||
thread_trace.record_protocol_event(&EventMsg::ShutdownComplete);
|
||||
|
||||
thread_trace.flush()?;
|
||||
let event_log = fs::read_to_string(single_bundle_dir(temp.path())?.join("trace.jsonl"))?;
|
||||
let protocol_event_seen = event_log.lines().any(|line| {
|
||||
let event: crate::RawTraceEvent = serde_json::from_str(line).expect("raw trace event");
|
||||
@@ -377,6 +389,7 @@ fn intercepted_exec_records_patch_without_a_terminal() -> anyhow::Result<()> {
|
||||
},
|
||||
}));
|
||||
let bundle = single_bundle_dir(temp.path())?;
|
||||
trace.flush()?;
|
||||
let before = replay_bundle(&bundle)?.tool_calls["patch-call"].clone();
|
||||
let changes = HashMap::from([(
|
||||
PathBuf::from("hello.txt"),
|
||||
@@ -415,6 +428,7 @@ fn intercepted_exec_records_patch_without_a_terminal() -> anyhow::Result<()> {
|
||||
},
|
||||
);
|
||||
|
||||
trace.flush()?;
|
||||
let rollout = replay_bundle(&bundle)?;
|
||||
let call = &rollout.tool_calls["patch-call"];
|
||||
assert_eq!(call.kind, ToolCallKind::ApplyPatch);
|
||||
|
||||
@@ -14,7 +14,6 @@ use codex_protocol::models::SearchToolCallParams;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value as JsonValue;
|
||||
use serde_json::json;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::model::AgentThreadId;
|
||||
use crate::model::CodeModeRuntimeToolId;
|
||||
@@ -25,7 +24,6 @@ use crate::model::ToolCallId;
|
||||
use crate::model::ToolCallKind;
|
||||
use crate::model::ToolCallSummary;
|
||||
use crate::payload::RawPayloadKind;
|
||||
use crate::payload::RawPayloadRef;
|
||||
use crate::raw_event::RawToolCallRequester;
|
||||
use crate::raw_event::RawTraceEventContext;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
@@ -201,15 +199,16 @@ fn record_started(context: &EnabledToolDispatchTraceContext, invocation: ToolDis
|
||||
tool_namespace: tool_namespace.as_deref(),
|
||||
payload: &payload,
|
||||
};
|
||||
let request_payload =
|
||||
write_json_payload_best_effort(&context.writer, RawPayloadKind::ToolInvocation, &request);
|
||||
let (model_visible_call_id, code_mode_runtime_tool_id, requester) =
|
||||
requester_fields(invocation.requester);
|
||||
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::ToolCallStarted {
|
||||
tool_call_id: context.tool_call_id.clone(),
|
||||
let tool_call_id = context.tool_call_id.clone();
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolInvocation,
|
||||
&request,
|
||||
move |reference| RawTraceEventPayload::ToolCallStarted {
|
||||
tool_call_id,
|
||||
model_visible_call_id,
|
||||
code_mode_runtime_tool_id,
|
||||
requester,
|
||||
@@ -219,7 +218,7 @@ fn record_started(context: &EnabledToolDispatchTraceContext, invocation: ToolDis
|
||||
input_preview,
|
||||
output_preview: None,
|
||||
},
|
||||
invocation_payload: request_payload,
|
||||
invocation_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -340,45 +339,19 @@ fn append_tool_call_ended(
|
||||
status: ExecutionStatus,
|
||||
response: &DispatchedToolTraceResponse<'_>,
|
||||
) {
|
||||
let response_payload =
|
||||
write_json_payload_best_effort(&context.writer, RawPayloadKind::ToolResult, response);
|
||||
append_with_context_best_effort(
|
||||
context,
|
||||
RawTraceEventPayload::ToolCallEnded {
|
||||
tool_call_id: context.tool_call_id.clone(),
|
||||
let tool_call_id = context.tool_call_id.clone();
|
||||
let _ = context.writer.event_with_payload(
|
||||
RawTraceEventContext::turn(&context.thread_id, &context.codex_turn_id),
|
||||
RawPayloadKind::ToolResult,
|
||||
response,
|
||||
move |reference| RawTraceEventPayload::ToolCallEnded {
|
||||
tool_call_id,
|
||||
status,
|
||||
result_payload: response_payload,
|
||||
result_payload: Some(reference),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn write_json_payload_best_effort(
|
||||
writer: &TraceWriter,
|
||||
kind: RawPayloadKind,
|
||||
payload: &impl Serialize,
|
||||
) -> Option<RawPayloadRef> {
|
||||
match writer.write_json_payload(kind, payload) {
|
||||
Ok(payload_ref) => Some(payload_ref),
|
||||
Err(err) => {
|
||||
warn!("failed to write rollout trace payload: {err:#}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_with_context_best_effort(
|
||||
context: &EnabledToolDispatchTraceContext,
|
||||
payload: RawTraceEventPayload,
|
||||
) {
|
||||
let event_context = RawTraceEventContext {
|
||||
thread_id: Some(context.thread_id.clone()),
|
||||
codex_turn_id: Some(context.codex_turn_id.clone()),
|
||||
};
|
||||
if let Err(err) = context.writer.append_with_context(event_context, payload) {
|
||||
warn!("failed to append rollout trace event: {err:#}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,21 +1,22 @@
|
||||
//! Hot-path trace bundle writer.
|
||||
//!
|
||||
//! Writers serialize whole payloads before publishing the events that reference them.
|
||||
//! Closing the bundle seals the final shard before recording completion.
|
||||
//! A single worker publishes live events in order while model and tool work proceeds.
|
||||
//! Shutdown waits for the worker to drain accepted events and seal the bundle.
|
||||
|
||||
use std::fmt;
|
||||
use std::fs::File;
|
||||
use std::io::BufWriter;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::PoisonError;
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use anyhow::anyhow;
|
||||
use anyhow::bail;
|
||||
use serde::Serialize;
|
||||
|
||||
@@ -32,15 +33,16 @@ use crate::raw_event::RawTraceEventContext;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
use crate::shard::ShardWriter;
|
||||
|
||||
/// Serializes access to the event log and its ordered payload shards.
|
||||
pub struct TraceWriter {
|
||||
bundle: Mutex<BundleWriter>,
|
||||
}
|
||||
// Bound the number of retained prompts before their serialized sizes are known.
|
||||
const MAX_PENDING_JOBS: usize = 8;
|
||||
type WriteJob = Box<dyn FnOnce(&mut BundleWriter) + Send>;
|
||||
|
||||
impl fmt::Debug for TraceWriter {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("TraceWriter").finish_non_exhaustive()
|
||||
}
|
||||
/// Serializes trace writes in submission order. Explicit writes wait until their evidence is readable.
|
||||
#[derive(Debug)]
|
||||
pub struct TraceWriter {
|
||||
jobs: Option<mpsc::SyncSender<WriteJob>>,
|
||||
worker: Option<thread::JoinHandle<()>>,
|
||||
failure: Arc<OnceLock<String>>,
|
||||
}
|
||||
|
||||
/// Owns the bundle files until shutdown. Each body is flushed before its event is published.
|
||||
@@ -51,7 +53,7 @@ pub(crate) struct BundleWriter {
|
||||
next_seq: u64,
|
||||
next_payload_ordinal: u64,
|
||||
// Keep the first error so explicit reads and shutdown can detect an incomplete recording.
|
||||
failure: OnceLock<String>,
|
||||
failure: Arc<OnceLock<String>>,
|
||||
}
|
||||
|
||||
impl BundleWriter {
|
||||
@@ -90,7 +92,7 @@ impl BundleWriter {
|
||||
) -> Result<RawTraceEvent> {
|
||||
if matches!(payload, RawTraceEventPayload::RecordingCompleted) {
|
||||
self.shards.finish()?;
|
||||
// Timestamp the closing marker after the shard is sealed.
|
||||
// Closure is observed here, after draining earlier jobs.
|
||||
wall_time_unix_ms = unix_time_ms();
|
||||
}
|
||||
let event = RawTraceEvent {
|
||||
@@ -133,28 +135,50 @@ impl TraceWriter {
|
||||
std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)
|
||||
.with_context(|| format!("write trace manifest {}", manifest_path.display()))?;
|
||||
|
||||
let bundle = BundleWriter {
|
||||
let failure = Arc::new(OnceLock::new());
|
||||
let mut bundle = BundleWriter {
|
||||
rollout_id: manifest.rollout_id,
|
||||
events: BufWriter::new(event_log),
|
||||
shards: ShardWriter::new(payloads_dir),
|
||||
next_seq: 1,
|
||||
next_payload_ordinal: 1,
|
||||
failure: OnceLock::new(),
|
||||
failure: Arc::clone(&failure),
|
||||
};
|
||||
let (jobs, receiver) = mpsc::sync_channel::<WriteJob>(MAX_PENDING_JOBS);
|
||||
let worker = thread::Builder::new()
|
||||
.name("rollout-trace".into())
|
||||
.spawn(move || {
|
||||
for job in receiver {
|
||||
job(&mut bundle);
|
||||
}
|
||||
})
|
||||
.context("start rollout trace writer")?;
|
||||
Ok(Self {
|
||||
bundle: Mutex::new(bundle),
|
||||
jobs: Some(jobs),
|
||||
worker: Some(worker),
|
||||
failure,
|
||||
})
|
||||
}
|
||||
|
||||
fn call<T>(&self, write: impl FnOnce(&mut BundleWriter) -> Result<T>) -> Result<T> {
|
||||
self.bundle
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.record(write)
|
||||
fn submit(&self, job: impl FnOnce(&mut BundleWriter) + Send + 'static) -> Result<()> {
|
||||
check_recording(&self.failure)?;
|
||||
self.jobs
|
||||
.as_ref()
|
||||
.context("closed rollout trace writer")?
|
||||
.send(Box::new(job))
|
||||
.map_err(|_| anyhow!("rollout trace worker stopped"))
|
||||
}
|
||||
|
||||
pub(crate) fn check(&self) -> Result<()> {
|
||||
self.call(|_| Ok(()))
|
||||
/// A caller that needs a published reference or a live read barrier waits in FIFO order.
|
||||
fn call<T: Send + 'static>(
|
||||
&self,
|
||||
write: impl FnOnce(&mut BundleWriter) -> Result<T> + Send + 'static,
|
||||
) -> Result<T> {
|
||||
let (reply, result) = mpsc::channel();
|
||||
self.submit(move |bundle| {
|
||||
let _ = reply.send(bundle.record(write));
|
||||
})?;
|
||||
result.recv().context("rollout trace worker stopped")?
|
||||
}
|
||||
|
||||
/// Publishes a complete JSON body and returns its location in the bundle.
|
||||
@@ -163,7 +187,9 @@ impl TraceWriter {
|
||||
kind: RawPayloadKind,
|
||||
value: &impl Serialize,
|
||||
) -> Result<RawPayloadRef> {
|
||||
self.call(|bundle| bundle.write_body(kind, &serde_json::to_vec(value)?))
|
||||
check_recording(&self.failure)?;
|
||||
let body = serde_json::to_vec(value);
|
||||
self.call(move |bundle| bundle.write_body(kind, &body?))
|
||||
}
|
||||
|
||||
/// Appends one raw event with no extra envelope context.
|
||||
@@ -181,38 +207,64 @@ impl TraceWriter {
|
||||
self.call(move |bundle| bundle.append(context, payload, now))
|
||||
}
|
||||
|
||||
/// Writes related payloads and their event under the same lock.
|
||||
pub(crate) fn record_event(
|
||||
/// The worker writes this event's bodies before publishing its references.
|
||||
pub(crate) fn defer_event(
|
||||
&self,
|
||||
context: RawTraceEventContext,
|
||||
build: impl FnOnce(&mut BundleWriter) -> Result<RawTraceEventPayload>,
|
||||
build: impl FnOnce(&mut BundleWriter) -> Result<RawTraceEventPayload> + Send + 'static,
|
||||
) -> Result<()> {
|
||||
self.call(|bundle| {
|
||||
let payload = build(bundle)?;
|
||||
bundle.append(context, payload, unix_time_ms())?;
|
||||
Ok(())
|
||||
let now = unix_time_ms();
|
||||
self.submit(move |bundle| {
|
||||
let _ = bundle.record(|bundle| {
|
||||
let payload = build(bundle)?;
|
||||
bundle.append(context, payload, now)
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
/// Publishes a JSON body before the event that points to it.
|
||||
pub(crate) fn enqueue_event(
|
||||
&self,
|
||||
context: RawTraceEventContext,
|
||||
payload: RawTraceEventPayload,
|
||||
) -> Result<()> {
|
||||
self.defer_event(context, move |_| Ok(payload))
|
||||
}
|
||||
|
||||
/// Encodes a borrowed observation before returning. The worker handles its
|
||||
/// bytes or serialization error in order, preserving previously accepted evidence.
|
||||
pub(crate) fn event_with_payload(
|
||||
&self,
|
||||
context: RawTraceEventContext,
|
||||
kind: RawPayloadKind,
|
||||
value: &impl Serialize,
|
||||
event: impl FnOnce(RawPayloadRef) -> RawTraceEventPayload,
|
||||
event: impl FnOnce(RawPayloadRef) -> RawTraceEventPayload + Send + 'static,
|
||||
) -> Result<()> {
|
||||
self.record_event(context, |bundle| {
|
||||
let reference = bundle.write_body(kind, &serde_json::to_vec(value)?)?;
|
||||
check_recording(&self.failure)?;
|
||||
let body = serde_json::to_vec(value);
|
||||
self.defer_event(context, move |bundle| {
|
||||
let reference = bundle.write_body(kind, &body?)?;
|
||||
Ok(event(reference))
|
||||
})
|
||||
}
|
||||
|
||||
/// Waits for earlier records to become readable, or returns the first recording failure.
|
||||
pub fn flush(&self) -> Result<()> {
|
||||
self.call(|_| Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TraceWriter {
|
||||
fn drop(&mut self) {
|
||||
if !self.bundle.is_poisoned() && !std::thread::panicking() {
|
||||
let _ = self.append(RawTraceEventPayload::RecordingCompleted);
|
||||
// The closing event seals the shard after earlier jobs. Joining makes it visible on return.
|
||||
if !thread::panicking() {
|
||||
let _ = self.enqueue_event(
|
||||
RawTraceEventContext::default(),
|
||||
RawTraceEventPayload::RecordingCompleted,
|
||||
);
|
||||
}
|
||||
drop(self.jobs.take());
|
||||
if let Some(worker) = self.worker.take() {
|
||||
let _ = worker.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -355,7 +407,7 @@ mod tests {
|
||||
.is_err()
|
||||
);
|
||||
std::fs::create_dir(temp.path().join("payloads"))?;
|
||||
assert!(writer.check().is_err());
|
||||
assert!(writer.flush().is_err());
|
||||
assert!(
|
||||
writer
|
||||
.write_json_payload(RawPayloadKind::ProtocolEvent, &json!({}))
|
||||
@@ -387,7 +439,7 @@ mod tests {
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(writer.check().is_err());
|
||||
assert!(writer.flush().is_err());
|
||||
drop(writer);
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(temp.path().join("trace.jsonl"))?,
|
||||
|
||||
Reference in New Issue
Block a user