Bound rollout trace capture with configurable storage limits (#61896)

## What changed

- Add `rollout_trace.max_payload_bytes` and `rollout_trace.max_bytes`, defaulting to 64 MiB per body or separated value and 256 MiB per bundle. Root and child threads share the bundle budget.
- Record explicit truncations and omissions when limits are reached, reserve space for later events, and stop at the last complete event if the event log fills.
- Reuse identical prepared-prompt items and store inline images separately from message structure. Preserve exact JSON bytes for complete composite payloads and surviving content and links in partial captures.
- Let reducers handle partial payloads while retaining lifecycle information and reporting `closed_with_partial_payloads` for closed bundles with incomplete captures.

## Testing

Add coverage for payload reuse and byte-exact reconstruction, truncation, compressed bundle budgets, later events after omissions, readable prefixes after event-space exhaustion, and CLI capture-limit configuration and startup failures.

GitOrigin-RevId: be99a68da427414a7a12bf8209f232c1a0bbe545
This commit is contained in:
cassirer-openai
2026-11-16 22:26:31 +00:00
committed by copyberry
parent c0d47262f3
commit ef6ce8100a
23 changed files with 712 additions and 107 deletions
+32 -7
View File
@@ -34,7 +34,9 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> {
let codex = codex_utils_cargo_bin::cargo_bin("codex")?;
let home = TempDir::new()?;
let server = create_mock_responses_server_sequence(Vec::new()).await;
MockResponsesConfig::new(&server.uri()).write(home.path())?;
MockResponsesConfig::new(&server.uri())
.with_extra_config("[rollout_trace]\nmax_payload_bytes = 64")
.write(home.path())?;
let trace_root = home.path().join("traces");
std::fs::write(&trace_root, "file")?;
let mut app = TestAppServer::builder()
@@ -61,16 +63,39 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> {
std::fs::remove_file(&trace_root)?;
std::fs::create_dir(&trace_root)?;
app.start_thread(ThreadStartParams::default()).await?;
assert!(app.shutdown_gracefully().await?.success());
let mut bundles = std::fs::read_dir(trace_root)?;
let bundle = bundles.next().unwrap()?.path();
assert!(bundles.next().is_none());
assert_eq!(
reduce_with_recording_binary(&codex, home.path(), &bundle)
.await?
.recording_status,
codex_rollout_trace::RecordingStatus::Complete
// A separate root with a budget too small for its manifest must fail at startup.
let request_id = app
.send_thread_start_request_with_auto_env(ThreadStartParams {
config: Some([("rollout_trace".into(), json!({"max_bytes": 1}))].into()),
..Default::default()
})
.await?;
let error = app
.read_stream_until_error_message(RequestId::Integer(request_id))
.await?;
assert!(
error
.error
.message
.contains("manifest exceeds trace byte limit")
);
assert!(app.shutdown_gracefully().await?.success());
let trace = reduce_with_recording_binary(&codex, home.path(), &bundle).await?;
assert_eq!(
trace.recording_status,
codex_rollout_trace::RecordingStatus::ClosedWithPartialPayloads
);
assert!(trace.raw_payloads.values().any(|p| matches!(
p.capture,
codex_rollout_trace::PayloadCapture::Truncated {
retained_bytes: 64,
..
}
)));
Ok(())
}
+13
View File
@@ -161,6 +161,16 @@ pub struct FeatureToggleToml {
pub enabled: Option<bool>,
}
/// Storage budgets for one native rollout trace, shared by the root and its children.
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)]
#[schemars(deny_unknown_fields)]
pub struct RolloutTraceToml {
/// Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB.
pub max_payload_bytes: Option<usize>,
/// Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB.
pub max_bytes: Option<u64>,
}
/// Base config deserialized from ~/.codex/config.toml.
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, JsonSchema)]
#[schemars(deny_unknown_fields)]
@@ -371,6 +381,9 @@ pub struct ConfigToml {
#[serde(default = "default_history")]
pub history: Option<History>,
/// Capture limits for local rollout traces when recording is enabled.
pub rollout_trace: Option<RolloutTraceToml>,
/// Directory where Codex stores the SQLite state DB.
/// Defaults to `$CODEX_SQLITE_HOME` when set. Otherwise uses `$CODEX_HOME`.
pub sqlite_home: Option<AbsolutePathBuf>,
+27
View File
@@ -4860,6 +4860,25 @@
},
"type": "object"
},
"RolloutTraceToml": {
"additionalProperties": false,
"description": "Storage budgets for one native rollout trace, shared by the root and its children.",
"properties": {
"max_bytes": {
"description": "Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB.",
"format": "uint64",
"minimum": 0.0,
"type": "integer"
},
"max_payload_bytes": {
"description": "Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB.",
"format": "uint",
"minimum": 0.0,
"type": "integer"
}
},
"type": "object"
},
"RuntimeAgentInstancesConfigToml": {
"additionalProperties": false,
"properties": {
@@ -9131,6 +9150,14 @@
"description": "Review model override used by the `/review` feature.",
"type": "string"
},
"rollout_trace": {
"allOf": [
{
"$ref": "#/definitions/RolloutTraceToml"
}
],
"description": "Capture limits for local rollout traces when recording is enabled."
},
"sandbox_mode": {
"allOf": [
{
+10 -2
View File
@@ -73,6 +73,7 @@ use codex_protocol_test_support::uuid_v7_item_id_generator_for_test;
use codex_rollout_trace::ExecutionStatus;
use codex_rollout_trace::InferenceTraceAttempt;
use codex_rollout_trace::InferenceTraceContext;
use codex_rollout_trace::RawPayloadKind;
use codex_rollout_trace::RawTraceEventPayload;
use codex_rollout_trace::RolloutTrace;
use codex_rollout_trace::TraceWriter;
@@ -1510,6 +1511,15 @@ async fn replay_until_cancelled(temp: &TempDir) -> anyhow::Result<RolloutTrace>
.next()
.expect("inference should be reduced");
if inference.execution.status == ExecutionStatus::Cancelled {
// Cancellation publishes one partial response. Prepared prompts can index shared child records.
assert_eq!(
rollout
.raw_payloads
.values()
.filter(|p| p.kind == RawPayloadKind::InferenceResponse)
.count(),
1
);
return Ok(rollout);
}
tokio::time::sleep(Duration::from_millis(10)).await;
@@ -1811,7 +1821,6 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re
rollout.threads["thread-root"].conversation_item_ids,
inference.request_item_ids
);
assert_eq!(rollout.raw_payloads.len(), 3);
Ok(())
}
@@ -2084,7 +2093,6 @@ async fn dropped_backpressured_response_stream_traces_cancelled_partial_output()
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
assert_eq!(inference.response_item_ids.len(), 1);
assert_eq!(rollout.raw_payloads.len(), 3);
Ok(())
}
+11
View File
@@ -142,6 +142,7 @@ use codex_protocol::protocol::AskForApproval;
use codex_protocol::protocol::MultiAgentVersion;
use codex_protocol::protocol::SandboxPolicy;
use codex_rmcp_client::McpOAuthRefreshMode;
use codex_rollout_trace::TraceCaptureLimits;
use codex_sandboxing::SandboxType;
pub use codex_thread_store::ExtraConfig;
use codex_utils_absolute_path::AbsolutePathBuf;
@@ -1040,6 +1041,9 @@ pub struct Config {
/// Settings that govern if and what will be written to `~/.codex/history.jsonl`.
pub history: History,
/// Limits shared by all threads recording into one native rollout trace.
pub rollout_trace_limits: TraceCaptureLimits,
/// When true, session is not persisted on disk. Default to `false`
pub ephemeral: bool,
@@ -4153,6 +4157,12 @@ impl Config {
let allow_login_shell = cfg.allow_login_shell.unwrap_or(true);
let history = cfg.history.unwrap_or_default();
let rollout_trace = cfg.rollout_trace.unwrap_or_default();
let defaults = TraceCaptureLimits::default();
let rollout_trace_limits = TraceCaptureLimits {
max_payload_bytes: rollout_trace.max_payload_bytes.unwrap_or(defaults.max_payload_bytes),
max_bytes: rollout_trace.max_bytes.unwrap_or(defaults.max_bytes),
};
if multi_agent_v2.max_concurrent_threads_per_session == 0 {
return Err(std::io::Error::new(
@@ -4762,6 +4772,7 @@ impl Config {
application_network_policy: Default::default(),
application_auth_route_config: None,
history,
rollout_trace_limits,
ephemeral: ephemeral.unwrap_or_default(),
allow_persistent_dynamic_tool_updates: false,
extra_config: None,
+1 -1
View File
@@ -1904,7 +1904,7 @@ impl Session {
parent_rollout_thread_trace.start_child_thread_trace(trace_metadata)
}
} else {
ThreadTraceContext::start_root(trace_metadata)
ThreadTraceContext::start_root(trace_metadata, config.rollout_trace_limits)
};
let rollout_thread_trace = rollout_thread_trace.unwrap_or_else(|err| {
startup_error.get_or_insert_with(|| {
+1 -1
View File
@@ -18,7 +18,7 @@ codex-code-mode = { workspace = true }
codex-protocol = { workspace = true }
http = { workspace = true }
serde = { workspace = true, features = ["derive"] }
serde_json = { workspace = true }
serde_json = { workspace = true, features = ["raw_value"] }
sha2 = { workspace = true }
tracing = { workspace = true }
zstd = { workspace = true }
+24 -11
View File
@@ -110,9 +110,8 @@ writer. Disabled contexts accept the same calls and record nothing.
When tracing is enabled, a thread can start or resume only if its recorder
initializes successfully. A later write failure stops recording. Partial
bundles remain available for replay. The reducer sets `recording_status` to `complete` in
`state.json` when it reads the closing record written by the last trace handle,
including when the recorded task was interrupted.
bundles remain available for replay. The reduced `recording_status` is `unclosed` until the closing record arrives. It then becomes
`complete` or `closed_with_partial_payloads` according to the referenced captures.
## Bundle Layout
@@ -124,12 +123,26 @@ A trace bundle contains:
events, terminal output, compaction data, and protocol snapshots.
- `state.json`: reduced trace for consumers, written by `codex debug trace-reduce`.
Schema v3 stores whole JSON bodies in numbered Zstandard shards. Every reference in
`trace.jsonl` carries the payload ID and shard number. The reducer collects these
references in `state.json` under `raw_payloads`, so the same index serves every
payload kind. `PayloadReader::read` resolves a reference to JSON bytes and caches
the two most recently read shards. An oversized shard occupies the cache alone.
Reduce a bundle with the binary that recorded it.
Schema v3 indexes numbered Zstandard shards in `state.json` under `raw_payloads`.
Prepared prompts share exact input items. Inline images have independent references and shards,
so losing an attachment preserves its position, sibling text and tool links. A composite record
stores the literal JSON fragments around its ordered `parts`. Interleaving both reconstructs the
original bytes, including whitespace and escapes. Hashes cover each entire value before truncation.
`PayloadReader::read` returns the original bytes for complete bodies. A truncated body
records its original and retained lengths and can be inspected with `read_prefix`.
Configure the capture limits in `config.toml`. Both values are in bytes.
```toml
[rollout_trace]
max_payload_bytes = 67108864 # Default: 64 MiB per new body or separated value
max_bytes = 268435456 # Default: 256 MiB per bundle
```
The bundle budget covers the manifest, event log, layouts, and compressed payloads.
Space is reserved for later events carrying explicit omissions.
If even an event cannot fit, the log ends at the preceding complete event.
The optional reduced-state cache is generated separately. Reduce a bundle with its recording binary.
For an inference call, `raw_request_payload_id` resolves to the outgoing HTTP
body or WebSocket message, including its `previous_response_id` and incremental
@@ -150,9 +163,9 @@ and write the closing marker. A recording failure prevents the marker.
Each shard is one Zstandard frame (level 3, 8 MiB window, checksum enabled).
Each record contains a little-endian 16-bit ID length, a little-endian 64-bit
body length, the UTF-8 ID, then the JSON bytes. The writer flushes a record
retained body length, the UTF-8 ID, then the retained bytes. The writer flushes a record
before publishing its reference, and rotates before the next record would
exceed 32 MiB uncompressed. An oversized record occupies one shard. The final
exceed 8 MiB uncompressed. An oversized record occupies one shard. The final
frame is sealed before the writer publishes `RecordingCompleted`. A live bundle
can be reduced through its published records with the last frame still open.
+139
View File
@@ -0,0 +1,139 @@
#![cfg(test)]
use super::*;
use crate::PayloadReader;
use crate::reducer::test_support::ROOT_THREAD_ID;
use crate::replay_bundle;
use pretty_assertions::assert_eq;
use serde_json::json;
use tempfile::TempDir;
pub(crate) fn limited_trace(body: usize, bundle: u64) -> Result<(TempDir, TraceWriter)> {
let root = TempDir::new()?;
let writer = TraceWriter::create_with_limits(
root.path(),
"t".into(),
"r".into(),
ROOT_THREAD_ID.into(),
TraceCaptureLimits {
max_payload_bytes: body,
max_bytes: bundle,
},
)?;
Ok((root, writer))
}
#[test]
fn parts_reuse_the_complete_identity_and_round_trip_in_order() -> Result<()> {
let (root, writer) = limited_trace(/*body*/ 80, /*bundle*/ 32 << 10)?;
let prefix = "a".repeat(80);
let oversized =
json!({"input": [format!("{prefix}1"), format!("{prefix}2"), format!("{prefix}1")]});
let first = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?;
let repeated = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?;
let items = &first.parts;
assert_eq!(first.parts, repeated.parts);
assert_eq!(items[0], items[2]);
assert_ne!(items[0].raw_payload_id, items[1].raw_payload_id);
assert_eq!(
items[0].capture,
PayloadCapture::Truncated {
original_bytes: 83,
retained_bytes: 80
}
);
let mut reader = PayloadReader::new(root.path());
assert!(reader.read(&first).is_err());
let captured = reader.read_prefix(&items[0])?.to_vec();
assert_eq!(captured, reader.read_prefix(&items[1])?);
let body = r#"{"z":true,"input":[{"s":"\u0061"},1.0,{"s":"\u0061"}],"a":[]}"#;
let prompt = writer.write_json_payload(
RawPayloadKind::PreparedPrompt,
&RawValue::from_string(body.to_string())?,
)?;
assert_eq!(reader.read(&prompt)?, body.as_bytes());
writer.append(RawTraceEventPayload::ProtocolEventObserved {
event_type: "test".into(),
event_payload: first,
})?;
drop(writer);
let state = replay_bundle(root.path())?;
assert_eq!(
state.recording_status,
crate::RecordingStatus::ClosedWithPartialPayloads
);
assert_eq!(state.raw_payloads.len(), 3);
Ok(())
}
#[test]
fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()> {
let limit = 4096;
let (root, writer) = limited_trace(/*body*/ 1 << 20, /*bundle*/ limit)?;
let compressible =
writer.write_json_payload(RawPayloadKind::ProtocolEvent, &"x".repeat(8192))?;
assert!(compressible.is_complete());
let noisy: String = (0u32..256)
.map(|i| format!("{:x}", Sha256::digest(i.to_le_bytes())))
.collect();
let omitted = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &noisy)?;
assert_eq!(
(&omitted.capture, omitted.shard),
(
&PayloadCapture::Omitted {
original_bytes: noisy.len() + 2
},
None
)
);
crate::reducer::test_support::start_thread(&writer, ROOT_THREAD_ID, "/root")?;
writer.append_with_context(
RawTraceEventContext::turn(ROOT_THREAD_ID, "turn"),
RawTraceEventPayload::HistoryItemAccepted {
item_id: "missing-receipt".into(),
agent_message_payload: Some(omitted.clone()),
},
)?;
let later = writer.write_json_payload(RawPayloadKind::ToolResult, &json!({"ok": true}))?;
for event_payload in [compressible, omitted, later.clone()] {
writer.append(RawTraceEventPayload::ProtocolEventObserved {
event_type: "test".into(),
event_payload,
})?;
}
drop(writer);
let state = replay_bundle(root.path())?;
assert_eq!(
state.recording_status,
crate::RecordingStatus::ClosedWithPartialPayloads
);
assert_eq!(
PayloadReader::new(root.path()).read_json(&later)?,
json!({"ok": true})
);
let mut bytes = std::fs::metadata(root.path().join("manifest.json"))?.len()
+ std::fs::metadata(root.path().join("trace.jsonl"))?.len();
for entry in std::fs::read_dir(root.path().join(PAYLOADS_DIR_NAME))? {
bytes += entry?.metadata()?.len();
}
assert!(bytes <= limit);
Ok(())
}
#[test]
fn exhausting_event_space_leaves_a_readable_prefix() -> Result<()> {
let (root, writer) = limited_trace(/*body*/ 1024, /*bundle*/ 1024)?;
let event_payload = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &true)?;
let record = |name: String| RawTraceEventPayload::ProtocolEventObserved {
event_type: name,
event_payload: event_payload.clone(),
};
writer.append(record("small".into()))?;
assert!(writer.append(record("x".repeat(1024))).is_err());
drop(writer);
let state = replay_bundle(root.path())?;
assert_eq!(state.recording_status, crate::RecordingStatus::Unclosed);
assert_eq!(state.raw_payloads.len(), 1);
Ok(())
}
+3 -1
View File
@@ -487,7 +487,9 @@ mod tests {
assert_eq!(inference.codex_turn_id, "turn-1");
assert_eq!(inference.execution.status, ExecutionStatus::Completed);
assert_eq!(inference.upstream_request_id, Some("req-1".to_string()));
assert_eq!(rollout.raw_payloads.len(), 3);
let prepared = &rollout.raw_payloads[&inference.prepared_prompt_payload_id];
assert_eq!(prepared.parts.len(), 1);
assert_eq!(rollout.raw_payloads.len(), 4);
Ok(())
}
+1
View File
@@ -79,5 +79,6 @@ pub use tool_dispatch::ToolDispatchRequester;
pub use tool_dispatch::ToolDispatchResult;
/// No-op-capable handle for recording one resolved tool dispatch.
pub use tool_dispatch::ToolDispatchTraceContext;
pub use writer::TraceCaptureLimits;
/// Append-only writer used by hot-path Codex instrumentation.
pub use writer::TraceWriter;
+76
View File
@@ -93,3 +93,79 @@ pub enum RawPayloadKind {
/// Runtime notification payload carried when a child agent reports back to its parent.
AgentResult,
}
/// Select whole JSON values whose bytes can be stored independently. Only model-facing
/// payloads use this layout. Other consumers can continue reading their single raw record.
pub(crate) fn capture_parts(
kind: RawPayloadKind,
body: &[u8],
limit: usize,
) -> anyhow::Result<Vec<(std::ops::Range<usize>, RawPayloadKind)>> {
use serde_json::value::RawValue;
use std::collections::BTreeMap;
if !matches!(
kind,
RawPayloadKind::PreparedPrompt
| RawPayloadKind::ModelItem
| RawPayloadKind::InferenceRequest
| RawPayloadKind::InferenceResponse
| RawPayloadKind::WarmupContext
| RawPayloadKind::CompactionCheckpoint
) {
return Ok(Vec::new());
}
let text = std::str::from_utf8(body)?;
if !matches!(text.trim_start().as_bytes().first(), Some(b'{' | b'['))
|| kind != RawPayloadKind::PreparedPrompt
&& text.len() <= limit
&& !text.contains("data:image/")
{
return Ok(Vec::new());
}
let mut pending = vec![(String::new(), serde_json::from_str::<&RawValue>(text)?)];
let mut ranges = Vec::new();
while let Some((field, raw)) = pending.pop() {
let value = raw.get();
let mut select = |value: &str, kind| {
let start = value.as_ptr() as usize - text.as_ptr() as usize;
ranges.push((start..start + value.len(), kind));
};
match value.as_bytes()[0] {
b'[' if (kind == RawPayloadKind::PreparedPrompt && field == "input")
|| (value.len() > limit
&& matches!(
field.as_str(),
"input" | "output_items" | "input_history" | "replacement_history"
)) =>
{
for item in serde_json::from_str::<Vec<&RawValue>>(value)? {
select(item.get(), RawPayloadKind::ModelItem);
}
}
b'"' if (field == "image_url" && value.starts_with("\"data:image/"))
|| (field == "text" && value.len() > limit) =>
{
select(value, RawPayloadKind::ContentValue);
}
b'{' | b'['
if value.len() > limit
|| value.contains("data:image/")
|| (kind == RawPayloadKind::PreparedPrompt && field.is_empty()) =>
{
if value.starts_with('{') {
pending.extend(serde_json::from_str::<BTreeMap<String, &RawValue>>(value)?);
} else {
pending.extend(
serde_json::from_str::<Vec<&RawValue>>(value)?
.into_iter()
.map(|item| ("[]".into(), item)),
);
}
}
_ => {}
}
}
ranges.sort_unstable_by_key(|(range, _)| range.start);
Ok(ranges)
}
@@ -74,7 +74,10 @@ impl TraceReducer {
codex_turn_id: &str,
request_payload: &RawPayloadRef,
) -> Result<Vec<String>> {
let payload = self.payload_reader.read_json(request_payload)?;
if !request_payload.capture.is_complete() {
return Ok(Vec::new());
}
let payload = self.payload_reader.observed_json(request_payload)?;
let Some(input) = payload.get("input") else {
bail!(
"inference request payload {} did not contain input",
@@ -117,9 +120,12 @@ impl TraceReducer {
.map(|inference| {
let mut ids = inference.request_item_ids.clone();
ids.extend(inference.response_item_ids.clone());
ids
let incomplete = std::iter::once(&inference.raw_request_payload_id)
.chain(inference.raw_response_payload_id.as_ref())
.any(|id| !self.rollout.raw_payloads[id].is_complete());
(ids, incomplete)
});
let Some(mut item_ids) = previous_items else {
let Some((mut item_ids, incomplete)) = previous_items else {
bail!(
"incremental inference request {inference_call_id} referenced unknown previous_response_id {previous_response_id}"
);
@@ -133,7 +139,9 @@ impl TraceReducer {
produced_by: Vec::new(),
start_index: item_ids.len(),
mode: ReconcileMode::AppendOnly,
snapshot_override: None,
// After a capture gap, the number of known prefix items is not
// a position in the thread's last fully observed snapshot.
snapshot_override: incomplete.then_some(item_ids.as_slice()),
},
)?;
item_ids.extend(delta_item_ids);
@@ -170,7 +178,10 @@ impl TraceReducer {
inference_call_id: &InferenceCallId,
response_payload: &RawPayloadRef,
) -> Result<Vec<String>> {
let payload = self.payload_reader.read_json(response_payload)?;
if !response_payload.capture.is_complete() {
return Ok(Vec::new());
}
let payload = self.payload_reader.observed_json(response_payload)?;
let Some(output_items) = payload.get("output_items").and_then(Value::as_array) else {
bail!(
"inference response payload {} did not contain output_items",
@@ -346,14 +357,18 @@ impl TraceReducer {
compaction_id: &CompactionId,
checkpoint_payload: &RawPayloadRef,
) -> Result<ReducedCompactionCheckpoint> {
let payload = self.payload_reader.read_json(checkpoint_payload)?;
let input_history = required_array(&payload, "input_history", checkpoint_payload)?;
let replacement_history =
required_array(&payload, "replacement_history", checkpoint_payload)?;
let input_items = normalize::normalize_model_items(input_history, checkpoint_payload)?;
let replacement_items =
normalize::normalize_model_items(replacement_history, checkpoint_payload)?;
let (input_items, replacement_items) = if checkpoint_payload.capture.is_complete() {
let payload = self.payload_reader.observed_json(checkpoint_payload)?;
let input = required_array(&payload, "input_history", checkpoint_payload)?;
let replacement = required_array(&payload, "replacement_history", checkpoint_payload)?;
(
normalize::normalize_model_items(input, checkpoint_payload)?,
normalize::normalize_model_items(replacement, checkpoint_payload)?,
)
} else {
// The install event still establishes a compaction boundary.
(Vec::new(), Vec::new())
};
let input_candidates = self
.thread_conversation_snapshots
.get(thread_id)
@@ -4,7 +4,7 @@ use anyhow::Context;
use anyhow::Result;
use anyhow::bail;
use codex_protocol::models::AgentMessageInputContent;
use codex_protocol::models::ResponseItem;
use serde::Deserialize;
use serde_json::Value;
use sha2::Digest;
use sha2::Sha256;
@@ -40,7 +40,9 @@ pub(super) fn normalize_model_items(
) -> Result<Vec<NormalizedConversationItem>> {
let mut normalized_items = Vec::new();
for item in items {
if item.get("type").and_then(Value::as_str) == Some("additional_tools") {
if item.get("type").and_then(Value::as_str) == Some("additional_tools")
|| unavailable_ref(item, raw_payload).is_some()
{
continue;
}
@@ -208,22 +210,19 @@ fn normalize_agent_message_item(
raw_payload: &RawPayloadRef,
) -> Result<NormalizedConversationItem> {
let raw_payload_id = &raw_payload.raw_payload_id;
let response_item =
serde_json::from_value::<ResponseItem>(item.clone()).with_context(|| {
format!("failed to parse agent_message item in payload {raw_payload_id}")
})?;
let ResponseItem::AgentMessage {
author,
recipient,
content,
..
} = response_item
else {
bail!("item in payload {raw_payload_id} was not an agent_message");
};
let parts = content
.into_iter()
.map(|content| match content {
let routing = AgentMessageMetadata::deserialize(item).with_context(|| {
format!("failed to parse agent_message routing in payload {raw_payload_id}")
})?;
let mut parts = Vec::new();
for part in item["content"]
.as_array()
.context("expected agent_message content array")?
{
if let Some(missing) = unavailable_ref(&part["text"], raw_payload) {
parts.push(payload_ref_part("text", missing));
continue;
}
parts.push(match AgentMessageInputContent::deserialize(part)? {
AgentMessageInputContent::InputText { text } => ConversationPart::Text { text },
AgentMessageInputContent::EncryptedContent { encrypted_content } => {
ConversationPart::Encoded {
@@ -231,8 +230,8 @@ fn normalize_agent_message_item(
value: encrypted_content,
}
}
})
.collect::<Vec<_>>();
});
}
if parts.is_empty() {
bail!("agent_message item in payload {raw_payload_id} contained no content");
}
@@ -242,7 +241,7 @@ fn normalize_agent_message_item(
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Analysis),
kind: ConversationItemKind::Message,
agent_message: Some(AgentMessageMetadata { author, recipient }),
agent_message: Some(routing),
body: ConversationBody { parts },
call_id: None,
})
@@ -359,6 +358,10 @@ fn append_reasoning_parts(
}
};
if let Some(missing) = unavailable_ref(&content_item["text"], raw_payload) {
parts.push(payload_ref_part(expected_type, missing));
continue;
}
let Some(text) = content_item.get("text").and_then(Value::as_str) else {
bail!(
"reasoning item in payload {} had {expected_type} entry without string text",
@@ -400,6 +403,14 @@ fn channel_from_phase(phase: &str) -> Option<ConversationChannel> {
}
}
/// Only interpret recorder-produced gaps backed by a partial reference in this payload.
fn unavailable_ref<'a>(value: &Value, root: &'a RawPayloadRef) -> Option<&'a RawPayloadRef> {
let marker = value.as_object().filter(|obj| obj.len() == 1)?;
let id = marker.get("$trace_unavailable")?.as_str()?;
root.references()
.find(|r| r.raw_payload_id == id && !r.capture.is_complete())
}
fn content_parts(
content: Option<&Value>,
raw_payload: &RawPayloadRef,
@@ -416,6 +427,8 @@ fn content_parts(
parts.push(ConversationPart::Text {
text: text.to_string(),
});
} else if let Some(missing) = unavailable_ref(&part["text"], raw_payload) {
parts.push(payload_ref_part("text", missing));
}
}
other => {
@@ -427,7 +440,10 @@ fn content_parts(
serde_json::to_writer(&mut digest, &canonical)?;
parts.push(ConversationPart::PayloadRef {
label: other.unwrap_or("content").to_string(),
raw_payload_id: raw_payload.raw_payload_id.clone(),
raw_payload_id: unavailable_ref(&part["image_url"], raw_payload)
.unwrap_or(raw_payload)
.raw_payload_id
.clone(),
content_sha256: Some(format!("{:x}", digest.finalize())),
});
}
@@ -15,11 +15,13 @@ use crate::model::ToolCallKind;
use crate::model::ToolCallSummary;
use crate::payload::RawPayloadKind;
use crate::raw_event::RawTraceEventPayload;
use crate::reducer::test_support::ROOT_THREAD_ID;
use crate::reducer::test_support::append_inference_completion;
use crate::reducer::test_support::append_inference_start;
use crate::reducer::test_support::create_started_writer;
use crate::reducer::test_support::expect_replay_error;
use crate::reducer::test_support::message;
use crate::reducer::test_support::start_thread;
use crate::reducer::test_support::start_turn;
use crate::reducer::test_support::trace_context;
use crate::replay_bundle;
@@ -1373,17 +1375,31 @@ fn context_compaction_boundary_repeats_prefix_and_reuses_replacement_items() ->
#[test]
fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Result<()> {
let temp = TempDir::new()?;
let writer = create_started_writer(&temp)?;
late_tool_start_with_image_limit(/*body_limit*/ 2048)
}
#[test]
fn missing_image_preserves_message_order_and_late_tool_links() -> anyhow::Result<()> {
late_tool_start_with_image_limit(/*body_limit*/ 512)
}
fn late_tool_start_with_image_limit(body_limit: usize) -> anyhow::Result<()> {
let (temp, writer) =
crate::writer::capture_tests::limited_trace(body_limit, /*bundle*/ 64 << 10)?;
start_thread(&writer, ROOT_THREAD_ID, "/root")?;
start_turn(&writer, "turn-1")?;
let request = writer.write_json_payload(
RawPayloadKind::InferenceRequest,
&json!({
// Cross the 9/10 item-ID boundary so lexical order differs from history order.
"input": vec![message("user", "run tests"); 7]
}),
)?;
// Cross the 9/10 item-ID boundary; one photo may be missing, but its message
// and every subsequent request, tool and output retain their original positions.
let mut input = vec![message("user", "run tests"); 6];
input.push(
json!({"type":"message", "id":"photo", "role":"user", "content":[
{"type":"input_text","text":"before"},
{"type":"input_image","image_url":format!("data:image/png;base64,{}", "x".repeat(1024))},
{"type":"input_text","text":"after"}]}),
);
let request =
writer.write_json_payload(RawPayloadKind::InferenceRequest, &json!({"input":input}))?;
append_inference_start(&writer, "inference-1", "turn-1", request)?;
let response = writer.write_json_payload(
@@ -1456,10 +1472,30 @@ fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Re
},
)?;
drop(writer);
let rollout = replay_bundle(temp.path())?;
let first_inference = &rollout.inference_calls["inference-1"];
let second_inference = &rollout.inference_calls["inference-2"];
let tool_call = &rollout.tool_calls["tool-1"];
let photo = &rollout.conversation_items[&first_inference.request_item_ids[6]];
assert_eq!(photo.responses_item_id.as_deref(), Some("photo"));
let [
ConversationPart::Text { text: before },
ConversationPart::PayloadRef { raw_payload_id, .. },
ConversationPart::Text { text: after },
] = photo.body.parts.as_slice()
else {
panic!("expected an image between its two text parts");
};
assert_eq!((before.as_str(), after.as_str()), ("before", "after"));
assert_eq!(
rollout.raw_payloads[raw_payload_id].is_complete(),
body_limit >= 1024
);
assert_eq!(
second_inference.request_item_ids[..7],
first_inference.request_item_ids
);
let output_item_ids = &second_inference.request_item_ids[8..];
assert_eq!(
+4 -1
View File
@@ -352,7 +352,10 @@ impl TraceReducer {
.thread_id
.context("history acceptance has no thread")?;
self.thread_mut(&thread_id)?;
if let Some(payload) = agent_message_payload {
// Keep the acceptance edge even when a capture limit hid the receipt body.
if let Some(payload) = agent_message_payload
&& payload.capture.is_complete()
{
self.record_agent_receipt(
&thread_id,
event.codex_turn_id,
@@ -42,6 +42,7 @@ impl TraceReducer {
let metadata = metadata_payload
.as_ref()
.filter(|p| p.capture.is_complete())
.map(|payload| self.thread_started_metadata(payload))
.transpose()?;
let spawn = metadata
@@ -79,6 +80,12 @@ impl TraceReducer {
task_name,
agent_role,
}
} else if thread_id != self.rollout.root_thread_id
&& metadata_payload
.as_ref()
.is_some_and(|p| !p.capture.is_complete())
{
AgentOrigin::Unknown
} else {
AgentOrigin::Root
};
@@ -72,6 +72,9 @@ impl TraceReducer {
tool_call_id: &str,
runtime_payload: &RawPayloadRef,
) -> Result<()> {
if !runtime_payload.capture.is_complete() {
return Ok(());
}
let kind = self
.rollout
.tool_calls
@@ -132,7 +135,7 @@ impl TraceReducer {
) -> Result<()> {
let kind = self.rollout.tool_calls[tool_call_id].kind.clone();
// MCP results are opaque evidence; they cannot describe agent interactions.
if matches!(kind, ToolCallKind::Mcp { .. }) {
if !runtime_payload.capture.is_complete() || matches!(kind, ToolCallKind::Mcp { .. }) {
return Ok(());
}
let runtime_payload_json = self.payload_reader.read_json(runtime_payload)?;
@@ -259,7 +262,10 @@ impl TraceReducer {
})?;
let started_at_unix_ms = tool_call.execution.started_at_unix_ms;
let message_author = self.agent_path_for_thread(&tool_call.thread_id)?;
let message_content = self.agent_message_content_from_invocation(tool_call_id)?;
let Some(message_content) = self.agent_message_content_from_invocation(tool_call_id)?
else {
return Ok(());
};
let carried_raw_payload_ids = self.agent_tool_payload_ids(tool_call_id)?;
self.queue_or_resolve_agent_interaction_edge(PendingAgentInteractionEdge {
edge_id,
@@ -277,7 +283,10 @@ impl TraceReducer {
})
}
fn agent_message_content_from_invocation(&mut self, tool_call_id: &str) -> Result<String> {
fn agent_message_content_from_invocation(
&mut self,
tool_call_id: &str,
) -> Result<Option<String>> {
let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| {
format!("agent activity referenced unknown tool call {tool_call_id}")
})?;
@@ -296,6 +305,9 @@ impl TraceReducer {
"agent activity tool call {tool_call_id} referenced missing invocation payload {invocation_payload_id}"
)
})?;
if !invocation_payload.capture.is_complete() {
return Ok(None);
}
let invocation = self.payload_reader.read_json(invocation_payload)?;
let arguments = invocation
.get("payload")
@@ -306,7 +318,7 @@ impl TraceReducer {
})?;
let args: AgentMessageInvocationArgs = serde_json::from_str(arguments)
.with_context(|| format!("parse agent activity tool call {tool_call_id} arguments"))?;
Ok(args.message)
Ok(Some(args.message))
}
/// Adds the canonical tool result payload to an already reduced multi-agent edge.
@@ -50,7 +50,8 @@ impl TraceReducer {
return Ok(None);
}
let operation_kind = TerminalOperationKind::WriteStdin;
let Some(invocation_payload) = invocation_payload else {
let Some(invocation_payload) = invocation_payload.filter(|p| p.capture.is_complete())
else {
// Payload writes are best-effort in the live recorder. If the
// canonical invocation is missing, keep the ToolCall but avoid
// fabricating a lossy terminal row.
@@ -92,6 +93,9 @@ impl TraceReducer {
return Ok(None);
};
if !runtime_payload.capture.is_complete() {
return Ok(None);
}
let payload = self.payload_reader.read_json(runtime_payload)?;
let payload: ExecCommandBeginPayload =
serde_json::from_value(payload).with_context(|| {
@@ -172,14 +176,10 @@ impl TraceReducer {
bail!("terminal end referenced unknown operation {operation_id}");
};
let response = response_payload
.filter(|p| p.capture.is_complete())
.map(|payload| {
let value = self.payload_reader.read_json(payload)?;
let response = parse_terminal_response_payload(
value,
&operation_kind,
&payload.raw_payload_id,
)?;
Ok::<_, anyhow::Error>((payload.raw_payload_id.clone(), response))
parse_terminal_response_payload(value, &operation_kind, &payload.raw_payload_id)
})
.transpose()?;
@@ -191,8 +191,10 @@ impl TraceReducer {
operation.execution.ended_seq = Some(seq);
operation.execution.status = status;
if let Some((raw_payload_id, response)) = response {
push_unique(&mut operation.raw_payload_ids, &raw_payload_id);
if let Some(payload) = response_payload {
push_unique(&mut operation.raw_payload_ids, &payload.raw_payload_id);
}
if let Some(response) = response {
operation.result = Some(response.result);
response.process_id
} else {
+66 -13
View File
@@ -1,4 +1,4 @@
//! Whole JSON bodies in ordered Zstandard shards. Each record is a little-endian
//! Retained payload bytes in ordered Zstandard shards. Each record is a little-endian
//! u16 ID length and u64 body length, followed by the UTF-8 ID and the body bytes.
//! A flush publishes one record without resetting the compression window.
@@ -24,8 +24,9 @@ use crate::payload::PayloadCapture;
use crate::payload::RawPayloadId;
use crate::payload::RawPayloadRef;
const TARGET_BYTES: usize = 32 * 1024 * 1024;
const WINDOW_LOG: u32 = 23; // 8 MiB of match history across payloads.
const WINDOW_LOG: u32 = 23;
// One match window per shard lets a 16 MiB hosted export admit whole media shards.
const TARGET_BYTES: usize = 1 << WINDOW_LOG;
const HEADER_BYTES: usize = 10;
fn shard_path(directory: &Path, number: u64) -> PathBuf {
@@ -37,6 +38,8 @@ pub(crate) struct ShardWriter {
active: Option<Encoder<'static, BufWriter<File>>>,
number: u64,
bytes: usize,
closed_bytes: u64,
active_bytes: u64,
}
impl ShardWriter {
@@ -46,12 +49,49 @@ impl ShardWriter {
active: None,
number: 0,
bytes: 0,
closed_bytes: 0,
active_bytes: 0,
}
}
/// Reserve the final empty block and checksum while a flushed frame is still live.
pub(crate) fn charged_bytes(&self) -> u64 {
self.closed_bytes + self.active_bytes + if self.active.is_some() { 7 } else { 0 }
}
/// Near the budget boundary, trial-compress one independent frame. A body that
/// does not fit then leaves both the published prefix and future small writes intact.
pub(crate) fn append_with_limit(
&mut self,
id: &str,
body: &[u8],
limit: u64,
) -> Result<Option<u64>> {
let size = HEADER_BYTES + id.len() + body.len();
let bound = zstd::zstd_safe::compress_bound(size) as u64 + 128;
if self.charged_bytes().saturating_add(bound) <= limit {
return self.append(id, body).map(Some);
}
self.finish()?;
if self.closed_bytes >= limit {
return Ok(None);
}
let mut encoder = new_encoder(Vec::new())?;
write_record(&mut encoder, id, body)?;
let frame = encoder.finish()?;
if frame.len() as u64 > limit - self.closed_bytes {
return Ok(None);
}
let number = self.number + 1;
let mut file = File::create_new(shard_path(&self.directory, number))?;
file.write_all(&frame)?;
self.closed_bytes += frame.len() as u64;
self.number = number;
Ok(Some(number))
}
/// Returns the shard containing this whole record once it is readable.
pub(crate) fn append(&mut self, id: &str, body: &[u8]) -> Result<u64> {
let id_len = u16::try_from(id.len())?;
let size = HEADER_BYTES + id.len() + body.len();
if self.bytes.saturating_add(size) > TARGET_BYTES {
self.finish()?;
@@ -64,17 +104,12 @@ impl ShardWriter {
let path = shard_path(&self.directory, self.number);
let file = File::create_new(&path)
.with_context(|| format!("create shard {}", path.display()))?;
let mut encoder = Encoder::new(BufWriter::new(file), 3)?;
encoder.window_log(WINDOW_LOG)?;
encoder.include_checksum(true)?;
self.active.insert(encoder)
self.active.insert(new_encoder(BufWriter::new(file))?)
}
};
encoder.write_all(&id_len.to_le_bytes())?;
encoder.write_all(&(body.len() as u64).to_le_bytes())?;
encoder.write_all(id.as_bytes())?;
encoder.write_all(body)?;
write_record(encoder, id, body)?;
encoder.flush()?;
self.active_bytes = encoder.get_ref().get_ref().metadata()?.len();
self.bytes += size;
Ok(self.number)
}
@@ -82,12 +117,30 @@ impl ShardWriter {
/// Flushes the frame footer before the event log can publish RecordingCompleted.
pub(crate) fn finish(&mut self) -> Result<()> {
if let Some(active) = self.active.take() {
active.finish()?.flush()?;
let mut file = active.finish()?;
file.flush()?;
self.closed_bytes += file.get_ref().metadata()?.len();
self.active_bytes = 0;
}
Ok(())
}
}
fn new_encoder<W: Write>(sink: W) -> Result<Encoder<'static, W>> {
let mut encoder = Encoder::new(sink, 3)?;
encoder.window_log(WINDOW_LOG)?;
encoder.include_checksum(true)?;
Ok(encoder)
}
fn write_record(sink: &mut impl Write, id: &str, body: &[u8]) -> Result<()> {
sink.write_all(&u16::try_from(id.len())?.to_le_bytes())?;
sink.write_all(&(body.len() as u64).to_le_bytes())?;
sink.write_all(id.as_bytes())?;
sink.write_all(body)?;
Ok(())
}
struct DecodedShard {
number: u64,
sealed: bool,
+10 -4
View File
@@ -33,6 +33,7 @@ use crate::RolloutStatus;
use crate::ToolCallId;
use crate::ToolDispatchInvocation;
use crate::ToolDispatchTraceContext;
use crate::TraceCaptureLimits;
use crate::TraceWriter;
use crate::protocol_event::codex_turn_trace_event;
use crate::protocol_event::tool_runtime_trace_event;
@@ -101,11 +102,14 @@ impl ThreadTraceContext {
}
/// Starts tracing when configured; an enabled recorder must initialize successfully.
pub fn start_root(metadata: ThreadStartedTraceMetadata) -> anyhow::Result<Self> {
pub fn start_root(
metadata: ThreadStartedTraceMetadata,
limits: TraceCaptureLimits,
) -> anyhow::Result<Self> {
let Some(root) = std::env::var_os(CODEX_ROLLOUT_TRACE_ROOT_ENV) else {
return Ok(Self::disabled());
};
start_root_in_root(&PathBuf::from(root), metadata)
start_root_in_root(&PathBuf::from(root), metadata, limits)
}
/// Starts a root trace in a known directory.
@@ -116,7 +120,7 @@ impl ThreadTraceContext {
root: &Path,
metadata: ThreadStartedTraceMetadata,
) -> anyhow::Result<Self> {
start_root_in_root(root, metadata)
start_root_in_root(root, metadata, TraceCaptureLimits::default())
}
/// Starts one thread lifecycle inside an existing rollout bundle.
@@ -551,15 +555,17 @@ impl ThreadTraceContext {
fn start_root_in_root(
root: &Path,
metadata: ThreadStartedTraceMetadata,
limits: TraceCaptureLimits,
) -> anyhow::Result<ThreadTraceContext> {
let trace_id = Uuid::new_v4().to_string();
let thread_id = metadata.thread_id.clone();
let bundle_dir = root.join(format!("trace-{trace_id}-{thread_id}"));
let writer = TraceWriter::create(
let writer = TraceWriter::create_with_limits(
&bundle_dir,
trace_id.clone(),
thread_id.clone(),
thread_id.clone(),
limits,
)?;
let writer = Arc::new(writer);
+148 -9
View File
@@ -3,6 +3,7 @@
//! A single worker publishes live events in order while model and tool work proceeds.
//! Shutdown waits for the worker to drain accepted events and seal the bundle.
use std::collections::HashMap;
use std::fs::File;
use std::io::BufWriter;
use std::io::Write;
@@ -14,11 +15,17 @@ use std::thread;
use std::time::SystemTime;
use std::time::UNIX_EPOCH;
use crate::payload::PayloadCapture;
use crate::payload::capture_parts;
use anyhow::Context;
use anyhow::Result;
use anyhow::anyhow;
use anyhow::bail;
use serde::Serialize;
#[cfg(test)]
use serde_json::value::RawValue;
use sha2::Digest;
use sha2::Sha256;
use crate::bundle::MANIFEST_FILE_NAME;
use crate::bundle::PAYLOADS_DIR_NAME;
@@ -37,6 +44,24 @@ use crate::shard::ShardWriter;
const MAX_PENDING_JOBS: usize = 8;
type WriteJob = Box<dyn FnOnce(&mut BundleWriter) + Send>;
/// Logical bytes per new body, and physical bytes for all files in one recording.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TraceCaptureLimits {
/// Maximum uncompressed bytes retained from a single body or separated value.
pub max_payload_bytes: usize,
/// Maximum bytes on disk for the manifest, event log, and compressed payload shards.
pub max_bytes: u64,
}
impl Default for TraceCaptureLimits {
fn default() -> Self {
Self {
max_payload_bytes: 64 << 20,
max_bytes: 256 << 20,
}
}
}
/// Serializes trace writes in submission order. Explicit writes wait until their evidence is readable.
#[derive(Debug)]
pub struct TraceWriter {
@@ -50,6 +75,11 @@ pub(crate) struct BundleWriter {
rollout_id: String,
events: BufWriter<File>,
shards: ShardWriter,
limits: TraceCaptureLimits,
metadata_bytes: u64,
event_log_full: bool,
items: HashMap<(RawPayloadKind, [u8; 32]), RawPayloadRef>,
writing_content: bool,
next_seq: u64,
next_payload_ordinal: u64,
// Keep the first error so explicit reads and shutdown can detect an incomplete recording.
@@ -59,7 +89,13 @@ pub(crate) struct BundleWriter {
impl BundleWriter {
fn record<T>(&mut self, write: impl FnOnce(&mut Self) -> Result<T>) -> Result<T> {
check_recording(&self.failure)?;
if self.event_log_full {
bail!("trace event log reached its byte limit");
}
write(self).inspect_err(|error| {
if self.event_log_full {
return;
}
let error = format!("{error:#}");
tracing::error!("rollout recording failed: {error}");
let _ = self.failure.set(error);
@@ -71,16 +107,81 @@ impl BundleWriter {
kind: RawPayloadKind,
body: &[u8],
) -> Result<RawPayloadRef> {
let parts = capture_parts(kind, body, self.limits.max_payload_bytes)?;
if parts.is_empty() {
return self.store_body(kind, body, body.len(), Vec::new());
}
// Literal fragments preserve whitespace, key order and escapes without reserving a
// sentinel in user JSON. A missing child occupies exactly one original value slot.
let mut fragments = Vec::with_capacity(parts.len() + 1);
let mut children = Vec::with_capacity(parts.len());
let mut end = 0;
for (range, child_kind) in parts {
fragments.push(std::str::from_utf8(&body[end..range.start])?);
end = range.end;
let bytes = &body[range];
let key = (child_kind, Sha256::digest(bytes).into());
let child = if let Some(child) = self.items.get(&key) {
child.clone()
} else {
let child = self.write_body(child_kind, bytes)?;
self.items.insert(key, child.clone());
child
};
children.push(child);
}
fragments.push(std::str::from_utf8(&body[end..])?);
self.store_body(kind, &serde_json::to_vec(&fragments)?, body.len(), children)
}
fn store_body(
&mut self,
kind: RawPayloadKind,
body: &[u8],
original_bytes: usize,
parts: Vec<RawPayloadRef>,
) -> Result<RawPayloadRef> {
// Bridge admits whole shards. Keep attachment bytes out of shards that carry
// message structure, so omitting a media shard cannot hide a tool or its links.
let content = kind == RawPayloadKind::ContentValue;
if self.writing_content != content {
self.shards.finish()?;
self.writing_content = content;
}
let ordinal = self.next_payload_ordinal;
let raw_payload_id = format!("raw_payload:{ordinal}");
let shard = self.shards.append(&raw_payload_id, body)?;
// Layout is metadata. Applying the individual body cap to it would hide surviving siblings.
let retained = if parts.is_empty() {
body.len().min(self.limits.max_payload_bytes)
} else {
body.len()
};
// Leave room for lifecycle events and their explicit omitted-body references.
let reserve = (self.limits.max_bytes / 8).min(1 << 20);
let shard_limit = self
.limits
.max_bytes
.saturating_sub(self.metadata_bytes + reserve + if content { reserve } else { 0 });
let shard =
self.shards
.append_with_limit(&raw_payload_id, &body[..retained], shard_limit)?;
let capture = if shard.is_none() {
PayloadCapture::Omitted { original_bytes }
} else if retained != body.len() {
PayloadCapture::Truncated {
original_bytes,
retained_bytes: retained,
}
} else {
PayloadCapture::Complete
};
self.next_payload_ordinal += 1;
Ok(RawPayloadRef {
raw_payload_id,
kind,
shard: Some(shard),
capture: crate::PayloadCapture::Complete,
parts: Vec::new(),
shard,
capture,
parts,
})
}
@@ -104,8 +205,16 @@ impl BundleWriter {
codex_turn_id: context.codex_turn_id,
payload,
};
serde_json::to_writer(&mut self.events, &event)?;
self.events.write_all(b"\n")?;
let mut bytes = serde_json::to_vec(&event)?;
bytes.push(b'\n');
if self.metadata_bytes + self.shards.charged_bytes() + bytes.len() as u64
> self.limits.max_bytes
{
self.event_log_full = true;
bail!("trace event log reached its byte limit");
}
self.events.write_all(&bytes)?;
self.metadata_bytes += bytes.len() as u64;
self.events.flush()?;
self.next_seq += 1;
Ok(event)
@@ -120,7 +229,22 @@ impl TraceWriter {
rollout_id: String,
root_thread_id: AgentThreadId,
) -> Result<Self> {
let bundle_dir = bundle_dir.as_ref();
Self::create_with_limits(
bundle_dir.as_ref(),
trace_id,
rollout_id,
root_thread_id,
TraceCaptureLimits::default(),
)
}
pub(crate) fn create_with_limits(
bundle_dir: &Path,
trace_id: String,
rollout_id: String,
root_thread_id: AgentThreadId,
limits: TraceCaptureLimits,
) -> Result<Self> {
let payloads_dir = bundle_dir.join(PAYLOADS_DIR_NAME);
std::fs::create_dir_all(&payloads_dir)
.with_context(|| format!("create trace payload dir {}", payloads_dir.display()))?;
@@ -128,11 +252,17 @@ impl TraceWriter {
let started_at_unix_ms = unix_time_ms();
let manifest =
TraceBundleManifest::new(trace_id, rollout_id, root_thread_id, started_at_unix_ms);
let manifest_bytes = serde_json::to_vec_pretty(&manifest)?;
let metadata_bytes = manifest_bytes.len() as u64;
anyhow::ensure!(
metadata_bytes <= limits.max_bytes,
"manifest exceeds trace byte limit"
);
let event_log_path = bundle_dir.join(RAW_EVENT_LOG_FILE_NAME);
let event_log = File::create_new(&event_log_path)
.with_context(|| format!("open trace event log {}", event_log_path.display()))?;
let manifest_path = bundle_dir.join(MANIFEST_FILE_NAME);
std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)
std::fs::write(&manifest_path, &manifest_bytes)
.with_context(|| format!("write trace manifest {}", manifest_path.display()))?;
let failure = Arc::new(OnceLock::new());
@@ -140,6 +270,11 @@ impl TraceWriter {
rollout_id: manifest.rollout_id,
events: BufWriter::new(event_log),
shards: ShardWriter::new(payloads_dir),
limits,
metadata_bytes,
event_log_full: false,
items: HashMap::new(),
writing_content: false,
next_seq: 1,
next_payload_ordinal: 1,
failure: Arc::clone(&failure),
@@ -181,7 +316,7 @@ impl TraceWriter {
result.recv().context("rollout trace worker stopped")?
}
/// Publishes a complete JSON body and returns its location in the bundle.
/// Records a JSON body and returns its capture status and storage reference.
pub fn write_json_payload(
&self,
kind: RawPayloadKind,
@@ -448,3 +583,7 @@ mod tests {
Ok(())
}
}
#[cfg(test)]
#[path = "capture_tests.rs"]
pub(crate) mod capture_tests;
@@ -356,6 +356,7 @@ async fn new_config(
log_dir: codex_home.join("log").to_path_buf(),
codex_home,
history: History::default(),
rollout_trace_limits: Default::default(),
ephemeral: true,
allow_persistent_dynamic_tool_updates: false,
extra_config: None,