mirror of
https://github.com/openai/codex.git
synced 2026-11-17 22:10:37 +00:00
Bound rollout trace capture with configurable storage limits (#61896)
## What changed - Add `rollout_trace.max_payload_bytes` and `rollout_trace.max_bytes`, defaulting to 64 MiB per body or separated value and 256 MiB per bundle. Root and child threads share the bundle budget. - Record explicit truncations and omissions when limits are reached, reserve space for later events, and stop at the last complete event if the event log fills. - Reuse identical prepared-prompt items and store inline images separately from message structure. Preserve exact JSON bytes for complete composite payloads and surviving content and links in partial captures. - Let reducers handle partial payloads while retaining lifecycle information and reporting `closed_with_partial_payloads` for closed bundles with incomplete captures. ## Testing Add coverage for payload reuse and byte-exact reconstruction, truncation, compressed bundle budgets, later events after omissions, readable prefixes after event-space exhaustion, and CLI capture-limit configuration and startup failures. GitOrigin-RevId: be99a68da427414a7a12bf8209f232c1a0bbe545
This commit is contained in:
committed by
copyberry
parent
c0d47262f3
commit
ef6ce8100a
@@ -34,7 +34,9 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> {
|
||||
let codex = codex_utils_cargo_bin::cargo_bin("codex")?;
|
||||
let home = TempDir::new()?;
|
||||
let server = create_mock_responses_server_sequence(Vec::new()).await;
|
||||
MockResponsesConfig::new(&server.uri()).write(home.path())?;
|
||||
MockResponsesConfig::new(&server.uri())
|
||||
.with_extra_config("[rollout_trace]\nmax_payload_bytes = 64")
|
||||
.write(home.path())?;
|
||||
let trace_root = home.path().join("traces");
|
||||
std::fs::write(&trace_root, "file")?;
|
||||
let mut app = TestAppServer::builder()
|
||||
@@ -61,16 +63,39 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> {
|
||||
std::fs::remove_file(&trace_root)?;
|
||||
std::fs::create_dir(&trace_root)?;
|
||||
app.start_thread(ThreadStartParams::default()).await?;
|
||||
assert!(app.shutdown_gracefully().await?.success());
|
||||
let mut bundles = std::fs::read_dir(trace_root)?;
|
||||
let bundle = bundles.next().unwrap()?.path();
|
||||
assert!(bundles.next().is_none());
|
||||
assert_eq!(
|
||||
reduce_with_recording_binary(&codex, home.path(), &bundle)
|
||||
.await?
|
||||
.recording_status,
|
||||
codex_rollout_trace::RecordingStatus::Complete
|
||||
|
||||
// A separate root with a budget too small for its manifest must fail at startup.
|
||||
let request_id = app
|
||||
.send_thread_start_request_with_auto_env(ThreadStartParams {
|
||||
config: Some([("rollout_trace".into(), json!({"max_bytes": 1}))].into()),
|
||||
..Default::default()
|
||||
})
|
||||
.await?;
|
||||
let error = app
|
||||
.read_stream_until_error_message(RequestId::Integer(request_id))
|
||||
.await?;
|
||||
assert!(
|
||||
error
|
||||
.error
|
||||
.message
|
||||
.contains("manifest exceeds trace byte limit")
|
||||
);
|
||||
assert!(app.shutdown_gracefully().await?.success());
|
||||
let trace = reduce_with_recording_binary(&codex, home.path(), &bundle).await?;
|
||||
assert_eq!(
|
||||
trace.recording_status,
|
||||
codex_rollout_trace::RecordingStatus::ClosedWithPartialPayloads
|
||||
);
|
||||
assert!(trace.raw_payloads.values().any(|p| matches!(
|
||||
p.capture,
|
||||
codex_rollout_trace::PayloadCapture::Truncated {
|
||||
retained_bytes: 64,
|
||||
..
|
||||
}
|
||||
)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -161,6 +161,16 @@ pub struct FeatureToggleToml {
|
||||
pub enabled: Option<bool>,
|
||||
}
|
||||
|
||||
/// Storage budgets for one native rollout trace, shared by the root and its children.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
pub struct RolloutTraceToml {
|
||||
/// Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB.
|
||||
pub max_payload_bytes: Option<usize>,
|
||||
/// Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB.
|
||||
pub max_bytes: Option<u64>,
|
||||
}
|
||||
|
||||
/// Base config deserialized from ~/.codex/config.toml.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, JsonSchema)]
|
||||
#[schemars(deny_unknown_fields)]
|
||||
@@ -371,6 +381,9 @@ pub struct ConfigToml {
|
||||
#[serde(default = "default_history")]
|
||||
pub history: Option<History>,
|
||||
|
||||
/// Capture limits for local rollout traces when recording is enabled.
|
||||
pub rollout_trace: Option<RolloutTraceToml>,
|
||||
|
||||
/// Directory where Codex stores the SQLite state DB.
|
||||
/// Defaults to `$CODEX_SQLITE_HOME` when set. Otherwise uses `$CODEX_HOME`.
|
||||
pub sqlite_home: Option<AbsolutePathBuf>,
|
||||
|
||||
@@ -4860,6 +4860,25 @@
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"RolloutTraceToml": {
|
||||
"additionalProperties": false,
|
||||
"description": "Storage budgets for one native rollout trace, shared by the root and its children.",
|
||||
"properties": {
|
||||
"max_bytes": {
|
||||
"description": "Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB.",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
},
|
||||
"max_payload_bytes": {
|
||||
"description": "Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB.",
|
||||
"format": "uint",
|
||||
"minimum": 0.0,
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
"type": "object"
|
||||
},
|
||||
"RuntimeAgentInstancesConfigToml": {
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
@@ -9131,6 +9150,14 @@
|
||||
"description": "Review model override used by the `/review` feature.",
|
||||
"type": "string"
|
||||
},
|
||||
"rollout_trace": {
|
||||
"allOf": [
|
||||
{
|
||||
"$ref": "#/definitions/RolloutTraceToml"
|
||||
}
|
||||
],
|
||||
"description": "Capture limits for local rollout traces when recording is enabled."
|
||||
},
|
||||
"sandbox_mode": {
|
||||
"allOf": [
|
||||
{
|
||||
|
||||
@@ -73,6 +73,7 @@ use codex_protocol_test_support::uuid_v7_item_id_generator_for_test;
|
||||
use codex_rollout_trace::ExecutionStatus;
|
||||
use codex_rollout_trace::InferenceTraceAttempt;
|
||||
use codex_rollout_trace::InferenceTraceContext;
|
||||
use codex_rollout_trace::RawPayloadKind;
|
||||
use codex_rollout_trace::RawTraceEventPayload;
|
||||
use codex_rollout_trace::RolloutTrace;
|
||||
use codex_rollout_trace::TraceWriter;
|
||||
@@ -1510,6 +1511,15 @@ async fn replay_until_cancelled(temp: &TempDir) -> anyhow::Result<RolloutTrace>
|
||||
.next()
|
||||
.expect("inference should be reduced");
|
||||
if inference.execution.status == ExecutionStatus::Cancelled {
|
||||
// Cancellation publishes one partial response. Prepared prompts can index shared child records.
|
||||
assert_eq!(
|
||||
rollout
|
||||
.raw_payloads
|
||||
.values()
|
||||
.filter(|p| p.kind == RawPayloadKind::InferenceResponse)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
return Ok(rollout);
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
@@ -1811,7 +1821,6 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re
|
||||
rollout.threads["thread-root"].conversation_item_ids,
|
||||
inference.request_item_ids
|
||||
);
|
||||
assert_eq!(rollout.raw_payloads.len(), 3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -2084,7 +2093,6 @@ async fn dropped_backpressured_response_stream_traces_cancelled_partial_output()
|
||||
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Cancelled);
|
||||
assert_eq!(inference.response_item_ids.len(), 1);
|
||||
assert_eq!(rollout.raw_payloads.len(), 3);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -142,6 +142,7 @@ use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::MultiAgentVersion;
|
||||
use codex_protocol::protocol::SandboxPolicy;
|
||||
use codex_rmcp_client::McpOAuthRefreshMode;
|
||||
use codex_rollout_trace::TraceCaptureLimits;
|
||||
use codex_sandboxing::SandboxType;
|
||||
pub use codex_thread_store::ExtraConfig;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
@@ -1040,6 +1041,9 @@ pub struct Config {
|
||||
/// Settings that govern if and what will be written to `~/.codex/history.jsonl`.
|
||||
pub history: History,
|
||||
|
||||
/// Limits shared by all threads recording into one native rollout trace.
|
||||
pub rollout_trace_limits: TraceCaptureLimits,
|
||||
|
||||
/// When true, session is not persisted on disk. Default to `false`
|
||||
pub ephemeral: bool,
|
||||
|
||||
@@ -4153,6 +4157,12 @@ impl Config {
|
||||
let allow_login_shell = cfg.allow_login_shell.unwrap_or(true);
|
||||
|
||||
let history = cfg.history.unwrap_or_default();
|
||||
let rollout_trace = cfg.rollout_trace.unwrap_or_default();
|
||||
let defaults = TraceCaptureLimits::default();
|
||||
let rollout_trace_limits = TraceCaptureLimits {
|
||||
max_payload_bytes: rollout_trace.max_payload_bytes.unwrap_or(defaults.max_payload_bytes),
|
||||
max_bytes: rollout_trace.max_bytes.unwrap_or(defaults.max_bytes),
|
||||
};
|
||||
|
||||
if multi_agent_v2.max_concurrent_threads_per_session == 0 {
|
||||
return Err(std::io::Error::new(
|
||||
@@ -4762,6 +4772,7 @@ impl Config {
|
||||
application_network_policy: Default::default(),
|
||||
application_auth_route_config: None,
|
||||
history,
|
||||
rollout_trace_limits,
|
||||
ephemeral: ephemeral.unwrap_or_default(),
|
||||
allow_persistent_dynamic_tool_updates: false,
|
||||
extra_config: None,
|
||||
|
||||
@@ -1904,7 +1904,7 @@ impl Session {
|
||||
parent_rollout_thread_trace.start_child_thread_trace(trace_metadata)
|
||||
}
|
||||
} else {
|
||||
ThreadTraceContext::start_root(trace_metadata)
|
||||
ThreadTraceContext::start_root(trace_metadata, config.rollout_trace_limits)
|
||||
};
|
||||
let rollout_thread_trace = rollout_thread_trace.unwrap_or_else(|err| {
|
||||
startup_error.get_or_insert_with(|| {
|
||||
|
||||
@@ -18,7 +18,7 @@ codex-code-mode = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
http = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
sha2 = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
zstd = { workspace = true }
|
||||
|
||||
@@ -110,9 +110,8 @@ writer. Disabled contexts accept the same calls and record nothing.
|
||||
|
||||
When tracing is enabled, a thread can start or resume only if its recorder
|
||||
initializes successfully. A later write failure stops recording. Partial
|
||||
bundles remain available for replay. The reducer sets `recording_status` to `complete` in
|
||||
`state.json` when it reads the closing record written by the last trace handle,
|
||||
including when the recorded task was interrupted.
|
||||
bundles remain available for replay. The reduced `recording_status` is `unclosed` until the closing record arrives. It then becomes
|
||||
`complete` or `closed_with_partial_payloads` according to the referenced captures.
|
||||
|
||||
## Bundle Layout
|
||||
|
||||
@@ -124,12 +123,26 @@ A trace bundle contains:
|
||||
events, terminal output, compaction data, and protocol snapshots.
|
||||
- `state.json`: reduced trace for consumers, written by `codex debug trace-reduce`.
|
||||
|
||||
Schema v3 stores whole JSON bodies in numbered Zstandard shards. Every reference in
|
||||
`trace.jsonl` carries the payload ID and shard number. The reducer collects these
|
||||
references in `state.json` under `raw_payloads`, so the same index serves every
|
||||
payload kind. `PayloadReader::read` resolves a reference to JSON bytes and caches
|
||||
the two most recently read shards. An oversized shard occupies the cache alone.
|
||||
Reduce a bundle with the binary that recorded it.
|
||||
Schema v3 indexes numbered Zstandard shards in `state.json` under `raw_payloads`.
|
||||
Prepared prompts share exact input items. Inline images have independent references and shards,
|
||||
so losing an attachment preserves its position, sibling text and tool links. A composite record
|
||||
stores the literal JSON fragments around its ordered `parts`. Interleaving both reconstructs the
|
||||
original bytes, including whitespace and escapes. Hashes cover each entire value before truncation.
|
||||
`PayloadReader::read` returns the original bytes for complete bodies. A truncated body
|
||||
records its original and retained lengths and can be inspected with `read_prefix`.
|
||||
|
||||
Configure the capture limits in `config.toml`. Both values are in bytes.
|
||||
|
||||
```toml
|
||||
[rollout_trace]
|
||||
max_payload_bytes = 67108864 # Default: 64 MiB per new body or separated value
|
||||
max_bytes = 268435456 # Default: 256 MiB per bundle
|
||||
```
|
||||
|
||||
The bundle budget covers the manifest, event log, layouts, and compressed payloads.
|
||||
Space is reserved for later events carrying explicit omissions.
|
||||
If even an event cannot fit, the log ends at the preceding complete event.
|
||||
The optional reduced-state cache is generated separately. Reduce a bundle with its recording binary.
|
||||
|
||||
For an inference call, `raw_request_payload_id` resolves to the outgoing HTTP
|
||||
body or WebSocket message, including its `previous_response_id` and incremental
|
||||
@@ -150,9 +163,9 @@ and write the closing marker. A recording failure prevents the marker.
|
||||
|
||||
Each shard is one Zstandard frame (level 3, 8 MiB window, checksum enabled).
|
||||
Each record contains a little-endian 16-bit ID length, a little-endian 64-bit
|
||||
body length, the UTF-8 ID, then the JSON bytes. The writer flushes a record
|
||||
retained body length, the UTF-8 ID, then the retained bytes. The writer flushes a record
|
||||
before publishing its reference, and rotates before the next record would
|
||||
exceed 32 MiB uncompressed. An oversized record occupies one shard. The final
|
||||
exceed 8 MiB uncompressed. An oversized record occupies one shard. The final
|
||||
frame is sealed before the writer publishes `RecordingCompleted`. A live bundle
|
||||
can be reduced through its published records with the last frame still open.
|
||||
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
#![cfg(test)]
|
||||
|
||||
use super::*;
|
||||
use crate::PayloadReader;
|
||||
use crate::reducer::test_support::ROOT_THREAD_ID;
|
||||
use crate::replay_bundle;
|
||||
use pretty_assertions::assert_eq;
|
||||
use serde_json::json;
|
||||
use tempfile::TempDir;
|
||||
|
||||
pub(crate) fn limited_trace(body: usize, bundle: u64) -> Result<(TempDir, TraceWriter)> {
|
||||
let root = TempDir::new()?;
|
||||
let writer = TraceWriter::create_with_limits(
|
||||
root.path(),
|
||||
"t".into(),
|
||||
"r".into(),
|
||||
ROOT_THREAD_ID.into(),
|
||||
TraceCaptureLimits {
|
||||
max_payload_bytes: body,
|
||||
max_bytes: bundle,
|
||||
},
|
||||
)?;
|
||||
Ok((root, writer))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parts_reuse_the_complete_identity_and_round_trip_in_order() -> Result<()> {
|
||||
let (root, writer) = limited_trace(/*body*/ 80, /*bundle*/ 32 << 10)?;
|
||||
let prefix = "a".repeat(80);
|
||||
let oversized =
|
||||
json!({"input": [format!("{prefix}1"), format!("{prefix}2"), format!("{prefix}1")]});
|
||||
let first = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?;
|
||||
let repeated = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?;
|
||||
let items = &first.parts;
|
||||
assert_eq!(first.parts, repeated.parts);
|
||||
assert_eq!(items[0], items[2]);
|
||||
assert_ne!(items[0].raw_payload_id, items[1].raw_payload_id);
|
||||
assert_eq!(
|
||||
items[0].capture,
|
||||
PayloadCapture::Truncated {
|
||||
original_bytes: 83,
|
||||
retained_bytes: 80
|
||||
}
|
||||
);
|
||||
let mut reader = PayloadReader::new(root.path());
|
||||
assert!(reader.read(&first).is_err());
|
||||
let captured = reader.read_prefix(&items[0])?.to_vec();
|
||||
assert_eq!(captured, reader.read_prefix(&items[1])?);
|
||||
|
||||
let body = r#"{"z":true,"input":[{"s":"\u0061"},1.0,{"s":"\u0061"}],"a":[]}"#;
|
||||
let prompt = writer.write_json_payload(
|
||||
RawPayloadKind::PreparedPrompt,
|
||||
&RawValue::from_string(body.to_string())?,
|
||||
)?;
|
||||
assert_eq!(reader.read(&prompt)?, body.as_bytes());
|
||||
writer.append(RawTraceEventPayload::ProtocolEventObserved {
|
||||
event_type: "test".into(),
|
||||
event_payload: first,
|
||||
})?;
|
||||
drop(writer);
|
||||
let state = replay_bundle(root.path())?;
|
||||
assert_eq!(
|
||||
state.recording_status,
|
||||
crate::RecordingStatus::ClosedWithPartialPayloads
|
||||
);
|
||||
assert_eq!(state.raw_payloads.len(), 3);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()> {
|
||||
let limit = 4096;
|
||||
let (root, writer) = limited_trace(/*body*/ 1 << 20, /*bundle*/ limit)?;
|
||||
let compressible =
|
||||
writer.write_json_payload(RawPayloadKind::ProtocolEvent, &"x".repeat(8192))?;
|
||||
assert!(compressible.is_complete());
|
||||
let noisy: String = (0u32..256)
|
||||
.map(|i| format!("{:x}", Sha256::digest(i.to_le_bytes())))
|
||||
.collect();
|
||||
let omitted = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &noisy)?;
|
||||
assert_eq!(
|
||||
(&omitted.capture, omitted.shard),
|
||||
(
|
||||
&PayloadCapture::Omitted {
|
||||
original_bytes: noisy.len() + 2
|
||||
},
|
||||
None
|
||||
)
|
||||
);
|
||||
crate::reducer::test_support::start_thread(&writer, ROOT_THREAD_ID, "/root")?;
|
||||
writer.append_with_context(
|
||||
RawTraceEventContext::turn(ROOT_THREAD_ID, "turn"),
|
||||
RawTraceEventPayload::HistoryItemAccepted {
|
||||
item_id: "missing-receipt".into(),
|
||||
agent_message_payload: Some(omitted.clone()),
|
||||
},
|
||||
)?;
|
||||
let later = writer.write_json_payload(RawPayloadKind::ToolResult, &json!({"ok": true}))?;
|
||||
for event_payload in [compressible, omitted, later.clone()] {
|
||||
writer.append(RawTraceEventPayload::ProtocolEventObserved {
|
||||
event_type: "test".into(),
|
||||
event_payload,
|
||||
})?;
|
||||
}
|
||||
drop(writer);
|
||||
let state = replay_bundle(root.path())?;
|
||||
assert_eq!(
|
||||
state.recording_status,
|
||||
crate::RecordingStatus::ClosedWithPartialPayloads
|
||||
);
|
||||
assert_eq!(
|
||||
PayloadReader::new(root.path()).read_json(&later)?,
|
||||
json!({"ok": true})
|
||||
);
|
||||
let mut bytes = std::fs::metadata(root.path().join("manifest.json"))?.len()
|
||||
+ std::fs::metadata(root.path().join("trace.jsonl"))?.len();
|
||||
for entry in std::fs::read_dir(root.path().join(PAYLOADS_DIR_NAME))? {
|
||||
bytes += entry?.metadata()?.len();
|
||||
}
|
||||
assert!(bytes <= limit);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exhausting_event_space_leaves_a_readable_prefix() -> Result<()> {
|
||||
let (root, writer) = limited_trace(/*body*/ 1024, /*bundle*/ 1024)?;
|
||||
let event_payload = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &true)?;
|
||||
let record = |name: String| RawTraceEventPayload::ProtocolEventObserved {
|
||||
event_type: name,
|
||||
event_payload: event_payload.clone(),
|
||||
};
|
||||
writer.append(record("small".into()))?;
|
||||
assert!(writer.append(record("x".repeat(1024))).is_err());
|
||||
drop(writer);
|
||||
let state = replay_bundle(root.path())?;
|
||||
assert_eq!(state.recording_status, crate::RecordingStatus::Unclosed);
|
||||
assert_eq!(state.raw_payloads.len(), 1);
|
||||
Ok(())
|
||||
}
|
||||
@@ -487,7 +487,9 @@ mod tests {
|
||||
assert_eq!(inference.codex_turn_id, "turn-1");
|
||||
assert_eq!(inference.execution.status, ExecutionStatus::Completed);
|
||||
assert_eq!(inference.upstream_request_id, Some("req-1".to_string()));
|
||||
assert_eq!(rollout.raw_payloads.len(), 3);
|
||||
let prepared = &rollout.raw_payloads[&inference.prepared_prompt_payload_id];
|
||||
assert_eq!(prepared.parts.len(), 1);
|
||||
assert_eq!(rollout.raw_payloads.len(), 4);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -79,5 +79,6 @@ pub use tool_dispatch::ToolDispatchRequester;
|
||||
pub use tool_dispatch::ToolDispatchResult;
|
||||
/// No-op-capable handle for recording one resolved tool dispatch.
|
||||
pub use tool_dispatch::ToolDispatchTraceContext;
|
||||
pub use writer::TraceCaptureLimits;
|
||||
/// Append-only writer used by hot-path Codex instrumentation.
|
||||
pub use writer::TraceWriter;
|
||||
|
||||
@@ -93,3 +93,79 @@ pub enum RawPayloadKind {
|
||||
/// Runtime notification payload carried when a child agent reports back to its parent.
|
||||
AgentResult,
|
||||
}
|
||||
|
||||
/// Select whole JSON values whose bytes can be stored independently. Only model-facing
|
||||
/// payloads use this layout. Other consumers can continue reading their single raw record.
|
||||
pub(crate) fn capture_parts(
|
||||
kind: RawPayloadKind,
|
||||
body: &[u8],
|
||||
limit: usize,
|
||||
) -> anyhow::Result<Vec<(std::ops::Range<usize>, RawPayloadKind)>> {
|
||||
use serde_json::value::RawValue;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
if !matches!(
|
||||
kind,
|
||||
RawPayloadKind::PreparedPrompt
|
||||
| RawPayloadKind::ModelItem
|
||||
| RawPayloadKind::InferenceRequest
|
||||
| RawPayloadKind::InferenceResponse
|
||||
| RawPayloadKind::WarmupContext
|
||||
| RawPayloadKind::CompactionCheckpoint
|
||||
) {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let text = std::str::from_utf8(body)?;
|
||||
if !matches!(text.trim_start().as_bytes().first(), Some(b'{' | b'['))
|
||||
|| kind != RawPayloadKind::PreparedPrompt
|
||||
&& text.len() <= limit
|
||||
&& !text.contains("data:image/")
|
||||
{
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut pending = vec![(String::new(), serde_json::from_str::<&RawValue>(text)?)];
|
||||
let mut ranges = Vec::new();
|
||||
while let Some((field, raw)) = pending.pop() {
|
||||
let value = raw.get();
|
||||
let mut select = |value: &str, kind| {
|
||||
let start = value.as_ptr() as usize - text.as_ptr() as usize;
|
||||
ranges.push((start..start + value.len(), kind));
|
||||
};
|
||||
match value.as_bytes()[0] {
|
||||
b'[' if (kind == RawPayloadKind::PreparedPrompt && field == "input")
|
||||
|| (value.len() > limit
|
||||
&& matches!(
|
||||
field.as_str(),
|
||||
"input" | "output_items" | "input_history" | "replacement_history"
|
||||
)) =>
|
||||
{
|
||||
for item in serde_json::from_str::<Vec<&RawValue>>(value)? {
|
||||
select(item.get(), RawPayloadKind::ModelItem);
|
||||
}
|
||||
}
|
||||
b'"' if (field == "image_url" && value.starts_with("\"data:image/"))
|
||||
|| (field == "text" && value.len() > limit) =>
|
||||
{
|
||||
select(value, RawPayloadKind::ContentValue);
|
||||
}
|
||||
b'{' | b'['
|
||||
if value.len() > limit
|
||||
|| value.contains("data:image/")
|
||||
|| (kind == RawPayloadKind::PreparedPrompt && field.is_empty()) =>
|
||||
{
|
||||
if value.starts_with('{') {
|
||||
pending.extend(serde_json::from_str::<BTreeMap<String, &RawValue>>(value)?);
|
||||
} else {
|
||||
pending.extend(
|
||||
serde_json::from_str::<Vec<&RawValue>>(value)?
|
||||
.into_iter()
|
||||
.map(|item| ("[]".into(), item)),
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
ranges.sort_unstable_by_key(|(range, _)| range.start);
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
@@ -74,7 +74,10 @@ impl TraceReducer {
|
||||
codex_turn_id: &str,
|
||||
request_payload: &RawPayloadRef,
|
||||
) -> Result<Vec<String>> {
|
||||
let payload = self.payload_reader.read_json(request_payload)?;
|
||||
if !request_payload.capture.is_complete() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let payload = self.payload_reader.observed_json(request_payload)?;
|
||||
let Some(input) = payload.get("input") else {
|
||||
bail!(
|
||||
"inference request payload {} did not contain input",
|
||||
@@ -117,9 +120,12 @@ impl TraceReducer {
|
||||
.map(|inference| {
|
||||
let mut ids = inference.request_item_ids.clone();
|
||||
ids.extend(inference.response_item_ids.clone());
|
||||
ids
|
||||
let incomplete = std::iter::once(&inference.raw_request_payload_id)
|
||||
.chain(inference.raw_response_payload_id.as_ref())
|
||||
.any(|id| !self.rollout.raw_payloads[id].is_complete());
|
||||
(ids, incomplete)
|
||||
});
|
||||
let Some(mut item_ids) = previous_items else {
|
||||
let Some((mut item_ids, incomplete)) = previous_items else {
|
||||
bail!(
|
||||
"incremental inference request {inference_call_id} referenced unknown previous_response_id {previous_response_id}"
|
||||
);
|
||||
@@ -133,7 +139,9 @@ impl TraceReducer {
|
||||
produced_by: Vec::new(),
|
||||
start_index: item_ids.len(),
|
||||
mode: ReconcileMode::AppendOnly,
|
||||
snapshot_override: None,
|
||||
// After a capture gap, the number of known prefix items is not
|
||||
// a position in the thread's last fully observed snapshot.
|
||||
snapshot_override: incomplete.then_some(item_ids.as_slice()),
|
||||
},
|
||||
)?;
|
||||
item_ids.extend(delta_item_ids);
|
||||
@@ -170,7 +178,10 @@ impl TraceReducer {
|
||||
inference_call_id: &InferenceCallId,
|
||||
response_payload: &RawPayloadRef,
|
||||
) -> Result<Vec<String>> {
|
||||
let payload = self.payload_reader.read_json(response_payload)?;
|
||||
if !response_payload.capture.is_complete() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let payload = self.payload_reader.observed_json(response_payload)?;
|
||||
let Some(output_items) = payload.get("output_items").and_then(Value::as_array) else {
|
||||
bail!(
|
||||
"inference response payload {} did not contain output_items",
|
||||
@@ -346,14 +357,18 @@ impl TraceReducer {
|
||||
compaction_id: &CompactionId,
|
||||
checkpoint_payload: &RawPayloadRef,
|
||||
) -> Result<ReducedCompactionCheckpoint> {
|
||||
let payload = self.payload_reader.read_json(checkpoint_payload)?;
|
||||
let input_history = required_array(&payload, "input_history", checkpoint_payload)?;
|
||||
let replacement_history =
|
||||
required_array(&payload, "replacement_history", checkpoint_payload)?;
|
||||
|
||||
let input_items = normalize::normalize_model_items(input_history, checkpoint_payload)?;
|
||||
let replacement_items =
|
||||
normalize::normalize_model_items(replacement_history, checkpoint_payload)?;
|
||||
let (input_items, replacement_items) = if checkpoint_payload.capture.is_complete() {
|
||||
let payload = self.payload_reader.observed_json(checkpoint_payload)?;
|
||||
let input = required_array(&payload, "input_history", checkpoint_payload)?;
|
||||
let replacement = required_array(&payload, "replacement_history", checkpoint_payload)?;
|
||||
(
|
||||
normalize::normalize_model_items(input, checkpoint_payload)?,
|
||||
normalize::normalize_model_items(replacement, checkpoint_payload)?,
|
||||
)
|
||||
} else {
|
||||
// The install event still establishes a compaction boundary.
|
||||
(Vec::new(), Vec::new())
|
||||
};
|
||||
let input_candidates = self
|
||||
.thread_conversation_snapshots
|
||||
.get(thread_id)
|
||||
|
||||
@@ -4,7 +4,7 @@ use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use anyhow::bail;
|
||||
use codex_protocol::models::AgentMessageInputContent;
|
||||
use codex_protocol::models::ResponseItem;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use sha2::Digest;
|
||||
use sha2::Sha256;
|
||||
@@ -40,7 +40,9 @@ pub(super) fn normalize_model_items(
|
||||
) -> Result<Vec<NormalizedConversationItem>> {
|
||||
let mut normalized_items = Vec::new();
|
||||
for item in items {
|
||||
if item.get("type").and_then(Value::as_str) == Some("additional_tools") {
|
||||
if item.get("type").and_then(Value::as_str) == Some("additional_tools")
|
||||
|| unavailable_ref(item, raw_payload).is_some()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -208,22 +210,19 @@ fn normalize_agent_message_item(
|
||||
raw_payload: &RawPayloadRef,
|
||||
) -> Result<NormalizedConversationItem> {
|
||||
let raw_payload_id = &raw_payload.raw_payload_id;
|
||||
let response_item =
|
||||
serde_json::from_value::<ResponseItem>(item.clone()).with_context(|| {
|
||||
format!("failed to parse agent_message item in payload {raw_payload_id}")
|
||||
})?;
|
||||
let ResponseItem::AgentMessage {
|
||||
author,
|
||||
recipient,
|
||||
content,
|
||||
..
|
||||
} = response_item
|
||||
else {
|
||||
bail!("item in payload {raw_payload_id} was not an agent_message");
|
||||
};
|
||||
let parts = content
|
||||
.into_iter()
|
||||
.map(|content| match content {
|
||||
let routing = AgentMessageMetadata::deserialize(item).with_context(|| {
|
||||
format!("failed to parse agent_message routing in payload {raw_payload_id}")
|
||||
})?;
|
||||
let mut parts = Vec::new();
|
||||
for part in item["content"]
|
||||
.as_array()
|
||||
.context("expected agent_message content array")?
|
||||
{
|
||||
if let Some(missing) = unavailable_ref(&part["text"], raw_payload) {
|
||||
parts.push(payload_ref_part("text", missing));
|
||||
continue;
|
||||
}
|
||||
parts.push(match AgentMessageInputContent::deserialize(part)? {
|
||||
AgentMessageInputContent::InputText { text } => ConversationPart::Text { text },
|
||||
AgentMessageInputContent::EncryptedContent { encrypted_content } => {
|
||||
ConversationPart::Encoded {
|
||||
@@ -231,8 +230,8 @@ fn normalize_agent_message_item(
|
||||
value: encrypted_content,
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
});
|
||||
}
|
||||
if parts.is_empty() {
|
||||
bail!("agent_message item in payload {raw_payload_id} contained no content");
|
||||
}
|
||||
@@ -242,7 +241,7 @@ fn normalize_agent_message_item(
|
||||
role: ConversationRole::Assistant,
|
||||
channel: Some(ConversationChannel::Analysis),
|
||||
kind: ConversationItemKind::Message,
|
||||
agent_message: Some(AgentMessageMetadata { author, recipient }),
|
||||
agent_message: Some(routing),
|
||||
body: ConversationBody { parts },
|
||||
call_id: None,
|
||||
})
|
||||
@@ -359,6 +358,10 @@ fn append_reasoning_parts(
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(missing) = unavailable_ref(&content_item["text"], raw_payload) {
|
||||
parts.push(payload_ref_part(expected_type, missing));
|
||||
continue;
|
||||
}
|
||||
let Some(text) = content_item.get("text").and_then(Value::as_str) else {
|
||||
bail!(
|
||||
"reasoning item in payload {} had {expected_type} entry without string text",
|
||||
@@ -400,6 +403,14 @@ fn channel_from_phase(phase: &str) -> Option<ConversationChannel> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Only interpret recorder-produced gaps backed by a partial reference in this payload.
|
||||
fn unavailable_ref<'a>(value: &Value, root: &'a RawPayloadRef) -> Option<&'a RawPayloadRef> {
|
||||
let marker = value.as_object().filter(|obj| obj.len() == 1)?;
|
||||
let id = marker.get("$trace_unavailable")?.as_str()?;
|
||||
root.references()
|
||||
.find(|r| r.raw_payload_id == id && !r.capture.is_complete())
|
||||
}
|
||||
|
||||
fn content_parts(
|
||||
content: Option<&Value>,
|
||||
raw_payload: &RawPayloadRef,
|
||||
@@ -416,6 +427,8 @@ fn content_parts(
|
||||
parts.push(ConversationPart::Text {
|
||||
text: text.to_string(),
|
||||
});
|
||||
} else if let Some(missing) = unavailable_ref(&part["text"], raw_payload) {
|
||||
parts.push(payload_ref_part("text", missing));
|
||||
}
|
||||
}
|
||||
other => {
|
||||
@@ -427,7 +440,10 @@ fn content_parts(
|
||||
serde_json::to_writer(&mut digest, &canonical)?;
|
||||
parts.push(ConversationPart::PayloadRef {
|
||||
label: other.unwrap_or("content").to_string(),
|
||||
raw_payload_id: raw_payload.raw_payload_id.clone(),
|
||||
raw_payload_id: unavailable_ref(&part["image_url"], raw_payload)
|
||||
.unwrap_or(raw_payload)
|
||||
.raw_payload_id
|
||||
.clone(),
|
||||
content_sha256: Some(format!("{:x}", digest.finalize())),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15,11 +15,13 @@ use crate::model::ToolCallKind;
|
||||
use crate::model::ToolCallSummary;
|
||||
use crate::payload::RawPayloadKind;
|
||||
use crate::raw_event::RawTraceEventPayload;
|
||||
use crate::reducer::test_support::ROOT_THREAD_ID;
|
||||
use crate::reducer::test_support::append_inference_completion;
|
||||
use crate::reducer::test_support::append_inference_start;
|
||||
use crate::reducer::test_support::create_started_writer;
|
||||
use crate::reducer::test_support::expect_replay_error;
|
||||
use crate::reducer::test_support::message;
|
||||
use crate::reducer::test_support::start_thread;
|
||||
use crate::reducer::test_support::start_turn;
|
||||
use crate::reducer::test_support::trace_context;
|
||||
use crate::replay_bundle;
|
||||
@@ -1373,17 +1375,31 @@ fn context_compaction_boundary_repeats_prefix_and_reuses_replacement_items() ->
|
||||
|
||||
#[test]
|
||||
fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Result<()> {
|
||||
let temp = TempDir::new()?;
|
||||
let writer = create_started_writer(&temp)?;
|
||||
late_tool_start_with_image_limit(/*body_limit*/ 2048)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_image_preserves_message_order_and_late_tool_links() -> anyhow::Result<()> {
|
||||
late_tool_start_with_image_limit(/*body_limit*/ 512)
|
||||
}
|
||||
|
||||
fn late_tool_start_with_image_limit(body_limit: usize) -> anyhow::Result<()> {
|
||||
let (temp, writer) =
|
||||
crate::writer::capture_tests::limited_trace(body_limit, /*bundle*/ 64 << 10)?;
|
||||
start_thread(&writer, ROOT_THREAD_ID, "/root")?;
|
||||
start_turn(&writer, "turn-1")?;
|
||||
|
||||
let request = writer.write_json_payload(
|
||||
RawPayloadKind::InferenceRequest,
|
||||
&json!({
|
||||
// Cross the 9/10 item-ID boundary so lexical order differs from history order.
|
||||
"input": vec![message("user", "run tests"); 7]
|
||||
}),
|
||||
)?;
|
||||
// Cross the 9/10 item-ID boundary; one photo may be missing, but its message
|
||||
// and every subsequent request, tool and output retain their original positions.
|
||||
let mut input = vec![message("user", "run tests"); 6];
|
||||
input.push(
|
||||
json!({"type":"message", "id":"photo", "role":"user", "content":[
|
||||
{"type":"input_text","text":"before"},
|
||||
{"type":"input_image","image_url":format!("data:image/png;base64,{}", "x".repeat(1024))},
|
||||
{"type":"input_text","text":"after"}]}),
|
||||
);
|
||||
let request =
|
||||
writer.write_json_payload(RawPayloadKind::InferenceRequest, &json!({"input":input}))?;
|
||||
append_inference_start(&writer, "inference-1", "turn-1", request)?;
|
||||
|
||||
let response = writer.write_json_payload(
|
||||
@@ -1456,10 +1472,30 @@ fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Re
|
||||
},
|
||||
)?;
|
||||
|
||||
drop(writer);
|
||||
let rollout = replay_bundle(temp.path())?;
|
||||
let first_inference = &rollout.inference_calls["inference-1"];
|
||||
let second_inference = &rollout.inference_calls["inference-2"];
|
||||
let tool_call = &rollout.tool_calls["tool-1"];
|
||||
let photo = &rollout.conversation_items[&first_inference.request_item_ids[6]];
|
||||
assert_eq!(photo.responses_item_id.as_deref(), Some("photo"));
|
||||
let [
|
||||
ConversationPart::Text { text: before },
|
||||
ConversationPart::PayloadRef { raw_payload_id, .. },
|
||||
ConversationPart::Text { text: after },
|
||||
] = photo.body.parts.as_slice()
|
||||
else {
|
||||
panic!("expected an image between its two text parts");
|
||||
};
|
||||
assert_eq!((before.as_str(), after.as_str()), ("before", "after"));
|
||||
assert_eq!(
|
||||
rollout.raw_payloads[raw_payload_id].is_complete(),
|
||||
body_limit >= 1024
|
||||
);
|
||||
assert_eq!(
|
||||
second_inference.request_item_ids[..7],
|
||||
first_inference.request_item_ids
|
||||
);
|
||||
let output_item_ids = &second_inference.request_item_ids[8..];
|
||||
|
||||
assert_eq!(
|
||||
|
||||
@@ -352,7 +352,10 @@ impl TraceReducer {
|
||||
.thread_id
|
||||
.context("history acceptance has no thread")?;
|
||||
self.thread_mut(&thread_id)?;
|
||||
if let Some(payload) = agent_message_payload {
|
||||
// Keep the acceptance edge even when a capture limit hid the receipt body.
|
||||
if let Some(payload) = agent_message_payload
|
||||
&& payload.capture.is_complete()
|
||||
{
|
||||
self.record_agent_receipt(
|
||||
&thread_id,
|
||||
event.codex_turn_id,
|
||||
|
||||
@@ -42,6 +42,7 @@ impl TraceReducer {
|
||||
|
||||
let metadata = metadata_payload
|
||||
.as_ref()
|
||||
.filter(|p| p.capture.is_complete())
|
||||
.map(|payload| self.thread_started_metadata(payload))
|
||||
.transpose()?;
|
||||
let spawn = metadata
|
||||
@@ -79,6 +80,12 @@ impl TraceReducer {
|
||||
task_name,
|
||||
agent_role,
|
||||
}
|
||||
} else if thread_id != self.rollout.root_thread_id
|
||||
&& metadata_payload
|
||||
.as_ref()
|
||||
.is_some_and(|p| !p.capture.is_complete())
|
||||
{
|
||||
AgentOrigin::Unknown
|
||||
} else {
|
||||
AgentOrigin::Root
|
||||
};
|
||||
|
||||
@@ -72,6 +72,9 @@ impl TraceReducer {
|
||||
tool_call_id: &str,
|
||||
runtime_payload: &RawPayloadRef,
|
||||
) -> Result<()> {
|
||||
if !runtime_payload.capture.is_complete() {
|
||||
return Ok(());
|
||||
}
|
||||
let kind = self
|
||||
.rollout
|
||||
.tool_calls
|
||||
@@ -132,7 +135,7 @@ impl TraceReducer {
|
||||
) -> Result<()> {
|
||||
let kind = self.rollout.tool_calls[tool_call_id].kind.clone();
|
||||
// MCP results are opaque evidence; they cannot describe agent interactions.
|
||||
if matches!(kind, ToolCallKind::Mcp { .. }) {
|
||||
if !runtime_payload.capture.is_complete() || matches!(kind, ToolCallKind::Mcp { .. }) {
|
||||
return Ok(());
|
||||
}
|
||||
let runtime_payload_json = self.payload_reader.read_json(runtime_payload)?;
|
||||
@@ -259,7 +262,10 @@ impl TraceReducer {
|
||||
})?;
|
||||
let started_at_unix_ms = tool_call.execution.started_at_unix_ms;
|
||||
let message_author = self.agent_path_for_thread(&tool_call.thread_id)?;
|
||||
let message_content = self.agent_message_content_from_invocation(tool_call_id)?;
|
||||
let Some(message_content) = self.agent_message_content_from_invocation(tool_call_id)?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let carried_raw_payload_ids = self.agent_tool_payload_ids(tool_call_id)?;
|
||||
self.queue_or_resolve_agent_interaction_edge(PendingAgentInteractionEdge {
|
||||
edge_id,
|
||||
@@ -277,7 +283,10 @@ impl TraceReducer {
|
||||
})
|
||||
}
|
||||
|
||||
fn agent_message_content_from_invocation(&mut self, tool_call_id: &str) -> Result<String> {
|
||||
fn agent_message_content_from_invocation(
|
||||
&mut self,
|
||||
tool_call_id: &str,
|
||||
) -> Result<Option<String>> {
|
||||
let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| {
|
||||
format!("agent activity referenced unknown tool call {tool_call_id}")
|
||||
})?;
|
||||
@@ -296,6 +305,9 @@ impl TraceReducer {
|
||||
"agent activity tool call {tool_call_id} referenced missing invocation payload {invocation_payload_id}"
|
||||
)
|
||||
})?;
|
||||
if !invocation_payload.capture.is_complete() {
|
||||
return Ok(None);
|
||||
}
|
||||
let invocation = self.payload_reader.read_json(invocation_payload)?;
|
||||
let arguments = invocation
|
||||
.get("payload")
|
||||
@@ -306,7 +318,7 @@ impl TraceReducer {
|
||||
})?;
|
||||
let args: AgentMessageInvocationArgs = serde_json::from_str(arguments)
|
||||
.with_context(|| format!("parse agent activity tool call {tool_call_id} arguments"))?;
|
||||
Ok(args.message)
|
||||
Ok(Some(args.message))
|
||||
}
|
||||
|
||||
/// Adds the canonical tool result payload to an already reduced multi-agent edge.
|
||||
|
||||
@@ -50,7 +50,8 @@ impl TraceReducer {
|
||||
return Ok(None);
|
||||
}
|
||||
let operation_kind = TerminalOperationKind::WriteStdin;
|
||||
let Some(invocation_payload) = invocation_payload else {
|
||||
let Some(invocation_payload) = invocation_payload.filter(|p| p.capture.is_complete())
|
||||
else {
|
||||
// Payload writes are best-effort in the live recorder. If the
|
||||
// canonical invocation is missing, keep the ToolCall but avoid
|
||||
// fabricating a lossy terminal row.
|
||||
@@ -92,6 +93,9 @@ impl TraceReducer {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if !runtime_payload.capture.is_complete() {
|
||||
return Ok(None);
|
||||
}
|
||||
let payload = self.payload_reader.read_json(runtime_payload)?;
|
||||
let payload: ExecCommandBeginPayload =
|
||||
serde_json::from_value(payload).with_context(|| {
|
||||
@@ -172,14 +176,10 @@ impl TraceReducer {
|
||||
bail!("terminal end referenced unknown operation {operation_id}");
|
||||
};
|
||||
let response = response_payload
|
||||
.filter(|p| p.capture.is_complete())
|
||||
.map(|payload| {
|
||||
let value = self.payload_reader.read_json(payload)?;
|
||||
let response = parse_terminal_response_payload(
|
||||
value,
|
||||
&operation_kind,
|
||||
&payload.raw_payload_id,
|
||||
)?;
|
||||
Ok::<_, anyhow::Error>((payload.raw_payload_id.clone(), response))
|
||||
parse_terminal_response_payload(value, &operation_kind, &payload.raw_payload_id)
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
@@ -191,8 +191,10 @@ impl TraceReducer {
|
||||
operation.execution.ended_seq = Some(seq);
|
||||
operation.execution.status = status;
|
||||
|
||||
if let Some((raw_payload_id, response)) = response {
|
||||
push_unique(&mut operation.raw_payload_ids, &raw_payload_id);
|
||||
if let Some(payload) = response_payload {
|
||||
push_unique(&mut operation.raw_payload_ids, &payload.raw_payload_id);
|
||||
}
|
||||
if let Some(response) = response {
|
||||
operation.result = Some(response.result);
|
||||
response.process_id
|
||||
} else {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Whole JSON bodies in ordered Zstandard shards. Each record is a little-endian
|
||||
//! Retained payload bytes in ordered Zstandard shards. Each record is a little-endian
|
||||
//! u16 ID length and u64 body length, followed by the UTF-8 ID and the body bytes.
|
||||
//! A flush publishes one record without resetting the compression window.
|
||||
|
||||
@@ -24,8 +24,9 @@ use crate::payload::PayloadCapture;
|
||||
use crate::payload::RawPayloadId;
|
||||
use crate::payload::RawPayloadRef;
|
||||
|
||||
const TARGET_BYTES: usize = 32 * 1024 * 1024;
|
||||
const WINDOW_LOG: u32 = 23; // 8 MiB of match history across payloads.
|
||||
const WINDOW_LOG: u32 = 23;
|
||||
// One match window per shard lets a 16 MiB hosted export admit whole media shards.
|
||||
const TARGET_BYTES: usize = 1 << WINDOW_LOG;
|
||||
const HEADER_BYTES: usize = 10;
|
||||
|
||||
fn shard_path(directory: &Path, number: u64) -> PathBuf {
|
||||
@@ -37,6 +38,8 @@ pub(crate) struct ShardWriter {
|
||||
active: Option<Encoder<'static, BufWriter<File>>>,
|
||||
number: u64,
|
||||
bytes: usize,
|
||||
closed_bytes: u64,
|
||||
active_bytes: u64,
|
||||
}
|
||||
|
||||
impl ShardWriter {
|
||||
@@ -46,12 +49,49 @@ impl ShardWriter {
|
||||
active: None,
|
||||
number: 0,
|
||||
bytes: 0,
|
||||
closed_bytes: 0,
|
||||
active_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Reserve the final empty block and checksum while a flushed frame is still live.
|
||||
pub(crate) fn charged_bytes(&self) -> u64 {
|
||||
self.closed_bytes + self.active_bytes + if self.active.is_some() { 7 } else { 0 }
|
||||
}
|
||||
|
||||
/// Near the budget boundary, trial-compress one independent frame. A body that
|
||||
/// does not fit then leaves both the published prefix and future small writes intact.
|
||||
pub(crate) fn append_with_limit(
|
||||
&mut self,
|
||||
id: &str,
|
||||
body: &[u8],
|
||||
limit: u64,
|
||||
) -> Result<Option<u64>> {
|
||||
let size = HEADER_BYTES + id.len() + body.len();
|
||||
let bound = zstd::zstd_safe::compress_bound(size) as u64 + 128;
|
||||
if self.charged_bytes().saturating_add(bound) <= limit {
|
||||
return self.append(id, body).map(Some);
|
||||
}
|
||||
self.finish()?;
|
||||
if self.closed_bytes >= limit {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut encoder = new_encoder(Vec::new())?;
|
||||
write_record(&mut encoder, id, body)?;
|
||||
let frame = encoder.finish()?;
|
||||
if frame.len() as u64 > limit - self.closed_bytes {
|
||||
return Ok(None);
|
||||
}
|
||||
let number = self.number + 1;
|
||||
let mut file = File::create_new(shard_path(&self.directory, number))?;
|
||||
file.write_all(&frame)?;
|
||||
self.closed_bytes += frame.len() as u64;
|
||||
self.number = number;
|
||||
Ok(Some(number))
|
||||
}
|
||||
|
||||
/// Returns the shard containing this whole record once it is readable.
|
||||
pub(crate) fn append(&mut self, id: &str, body: &[u8]) -> Result<u64> {
|
||||
let id_len = u16::try_from(id.len())?;
|
||||
let size = HEADER_BYTES + id.len() + body.len();
|
||||
if self.bytes.saturating_add(size) > TARGET_BYTES {
|
||||
self.finish()?;
|
||||
@@ -64,17 +104,12 @@ impl ShardWriter {
|
||||
let path = shard_path(&self.directory, self.number);
|
||||
let file = File::create_new(&path)
|
||||
.with_context(|| format!("create shard {}", path.display()))?;
|
||||
let mut encoder = Encoder::new(BufWriter::new(file), 3)?;
|
||||
encoder.window_log(WINDOW_LOG)?;
|
||||
encoder.include_checksum(true)?;
|
||||
self.active.insert(encoder)
|
||||
self.active.insert(new_encoder(BufWriter::new(file))?)
|
||||
}
|
||||
};
|
||||
encoder.write_all(&id_len.to_le_bytes())?;
|
||||
encoder.write_all(&(body.len() as u64).to_le_bytes())?;
|
||||
encoder.write_all(id.as_bytes())?;
|
||||
encoder.write_all(body)?;
|
||||
write_record(encoder, id, body)?;
|
||||
encoder.flush()?;
|
||||
self.active_bytes = encoder.get_ref().get_ref().metadata()?.len();
|
||||
self.bytes += size;
|
||||
Ok(self.number)
|
||||
}
|
||||
@@ -82,12 +117,30 @@ impl ShardWriter {
|
||||
/// Flushes the frame footer before the event log can publish RecordingCompleted.
|
||||
pub(crate) fn finish(&mut self) -> Result<()> {
|
||||
if let Some(active) = self.active.take() {
|
||||
active.finish()?.flush()?;
|
||||
let mut file = active.finish()?;
|
||||
file.flush()?;
|
||||
self.closed_bytes += file.get_ref().metadata()?.len();
|
||||
self.active_bytes = 0;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn new_encoder<W: Write>(sink: W) -> Result<Encoder<'static, W>> {
|
||||
let mut encoder = Encoder::new(sink, 3)?;
|
||||
encoder.window_log(WINDOW_LOG)?;
|
||||
encoder.include_checksum(true)?;
|
||||
Ok(encoder)
|
||||
}
|
||||
|
||||
fn write_record(sink: &mut impl Write, id: &str, body: &[u8]) -> Result<()> {
|
||||
sink.write_all(&u16::try_from(id.len())?.to_le_bytes())?;
|
||||
sink.write_all(&(body.len() as u64).to_le_bytes())?;
|
||||
sink.write_all(id.as_bytes())?;
|
||||
sink.write_all(body)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct DecodedShard {
|
||||
number: u64,
|
||||
sealed: bool,
|
||||
|
||||
@@ -33,6 +33,7 @@ use crate::RolloutStatus;
|
||||
use crate::ToolCallId;
|
||||
use crate::ToolDispatchInvocation;
|
||||
use crate::ToolDispatchTraceContext;
|
||||
use crate::TraceCaptureLimits;
|
||||
use crate::TraceWriter;
|
||||
use crate::protocol_event::codex_turn_trace_event;
|
||||
use crate::protocol_event::tool_runtime_trace_event;
|
||||
@@ -101,11 +102,14 @@ impl ThreadTraceContext {
|
||||
}
|
||||
|
||||
/// Starts tracing when configured; an enabled recorder must initialize successfully.
|
||||
pub fn start_root(metadata: ThreadStartedTraceMetadata) -> anyhow::Result<Self> {
|
||||
pub fn start_root(
|
||||
metadata: ThreadStartedTraceMetadata,
|
||||
limits: TraceCaptureLimits,
|
||||
) -> anyhow::Result<Self> {
|
||||
let Some(root) = std::env::var_os(CODEX_ROLLOUT_TRACE_ROOT_ENV) else {
|
||||
return Ok(Self::disabled());
|
||||
};
|
||||
start_root_in_root(&PathBuf::from(root), metadata)
|
||||
start_root_in_root(&PathBuf::from(root), metadata, limits)
|
||||
}
|
||||
|
||||
/// Starts a root trace in a known directory.
|
||||
@@ -116,7 +120,7 @@ impl ThreadTraceContext {
|
||||
root: &Path,
|
||||
metadata: ThreadStartedTraceMetadata,
|
||||
) -> anyhow::Result<Self> {
|
||||
start_root_in_root(root, metadata)
|
||||
start_root_in_root(root, metadata, TraceCaptureLimits::default())
|
||||
}
|
||||
|
||||
/// Starts one thread lifecycle inside an existing rollout bundle.
|
||||
@@ -551,15 +555,17 @@ impl ThreadTraceContext {
|
||||
fn start_root_in_root(
|
||||
root: &Path,
|
||||
metadata: ThreadStartedTraceMetadata,
|
||||
limits: TraceCaptureLimits,
|
||||
) -> anyhow::Result<ThreadTraceContext> {
|
||||
let trace_id = Uuid::new_v4().to_string();
|
||||
let thread_id = metadata.thread_id.clone();
|
||||
let bundle_dir = root.join(format!("trace-{trace_id}-{thread_id}"));
|
||||
let writer = TraceWriter::create(
|
||||
let writer = TraceWriter::create_with_limits(
|
||||
&bundle_dir,
|
||||
trace_id.clone(),
|
||||
thread_id.clone(),
|
||||
thread_id.clone(),
|
||||
limits,
|
||||
)?;
|
||||
let writer = Arc::new(writer);
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! A single worker publishes live events in order while model and tool work proceeds.
|
||||
//! Shutdown waits for the worker to drain accepted events and seal the bundle.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::BufWriter;
|
||||
use std::io::Write;
|
||||
@@ -14,11 +15,17 @@ use std::thread;
|
||||
use std::time::SystemTime;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use crate::payload::PayloadCapture;
|
||||
use crate::payload::capture_parts;
|
||||
use anyhow::Context;
|
||||
use anyhow::Result;
|
||||
use anyhow::anyhow;
|
||||
use anyhow::bail;
|
||||
use serde::Serialize;
|
||||
#[cfg(test)]
|
||||
use serde_json::value::RawValue;
|
||||
use sha2::Digest;
|
||||
use sha2::Sha256;
|
||||
|
||||
use crate::bundle::MANIFEST_FILE_NAME;
|
||||
use crate::bundle::PAYLOADS_DIR_NAME;
|
||||
@@ -37,6 +44,24 @@ use crate::shard::ShardWriter;
|
||||
const MAX_PENDING_JOBS: usize = 8;
|
||||
type WriteJob = Box<dyn FnOnce(&mut BundleWriter) + Send>;
|
||||
|
||||
/// Logical bytes per new body, and physical bytes for all files in one recording.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TraceCaptureLimits {
|
||||
/// Maximum uncompressed bytes retained from a single body or separated value.
|
||||
pub max_payload_bytes: usize,
|
||||
/// Maximum bytes on disk for the manifest, event log, and compressed payload shards.
|
||||
pub max_bytes: u64,
|
||||
}
|
||||
|
||||
impl Default for TraceCaptureLimits {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_payload_bytes: 64 << 20,
|
||||
max_bytes: 256 << 20,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes trace writes in submission order. Explicit writes wait until their evidence is readable.
|
||||
#[derive(Debug)]
|
||||
pub struct TraceWriter {
|
||||
@@ -50,6 +75,11 @@ pub(crate) struct BundleWriter {
|
||||
rollout_id: String,
|
||||
events: BufWriter<File>,
|
||||
shards: ShardWriter,
|
||||
limits: TraceCaptureLimits,
|
||||
metadata_bytes: u64,
|
||||
event_log_full: bool,
|
||||
items: HashMap<(RawPayloadKind, [u8; 32]), RawPayloadRef>,
|
||||
writing_content: bool,
|
||||
next_seq: u64,
|
||||
next_payload_ordinal: u64,
|
||||
// Keep the first error so explicit reads and shutdown can detect an incomplete recording.
|
||||
@@ -59,7 +89,13 @@ pub(crate) struct BundleWriter {
|
||||
impl BundleWriter {
|
||||
fn record<T>(&mut self, write: impl FnOnce(&mut Self) -> Result<T>) -> Result<T> {
|
||||
check_recording(&self.failure)?;
|
||||
if self.event_log_full {
|
||||
bail!("trace event log reached its byte limit");
|
||||
}
|
||||
write(self).inspect_err(|error| {
|
||||
if self.event_log_full {
|
||||
return;
|
||||
}
|
||||
let error = format!("{error:#}");
|
||||
tracing::error!("rollout recording failed: {error}");
|
||||
let _ = self.failure.set(error);
|
||||
@@ -71,16 +107,81 @@ impl BundleWriter {
|
||||
kind: RawPayloadKind,
|
||||
body: &[u8],
|
||||
) -> Result<RawPayloadRef> {
|
||||
let parts = capture_parts(kind, body, self.limits.max_payload_bytes)?;
|
||||
if parts.is_empty() {
|
||||
return self.store_body(kind, body, body.len(), Vec::new());
|
||||
}
|
||||
// Literal fragments preserve whitespace, key order and escapes without reserving a
|
||||
// sentinel in user JSON. A missing child occupies exactly one original value slot.
|
||||
let mut fragments = Vec::with_capacity(parts.len() + 1);
|
||||
let mut children = Vec::with_capacity(parts.len());
|
||||
let mut end = 0;
|
||||
for (range, child_kind) in parts {
|
||||
fragments.push(std::str::from_utf8(&body[end..range.start])?);
|
||||
end = range.end;
|
||||
let bytes = &body[range];
|
||||
let key = (child_kind, Sha256::digest(bytes).into());
|
||||
let child = if let Some(child) = self.items.get(&key) {
|
||||
child.clone()
|
||||
} else {
|
||||
let child = self.write_body(child_kind, bytes)?;
|
||||
self.items.insert(key, child.clone());
|
||||
child
|
||||
};
|
||||
children.push(child);
|
||||
}
|
||||
fragments.push(std::str::from_utf8(&body[end..])?);
|
||||
self.store_body(kind, &serde_json::to_vec(&fragments)?, body.len(), children)
|
||||
}
|
||||
|
||||
fn store_body(
|
||||
&mut self,
|
||||
kind: RawPayloadKind,
|
||||
body: &[u8],
|
||||
original_bytes: usize,
|
||||
parts: Vec<RawPayloadRef>,
|
||||
) -> Result<RawPayloadRef> {
|
||||
// Bridge admits whole shards. Keep attachment bytes out of shards that carry
|
||||
// message structure, so omitting a media shard cannot hide a tool or its links.
|
||||
let content = kind == RawPayloadKind::ContentValue;
|
||||
if self.writing_content != content {
|
||||
self.shards.finish()?;
|
||||
self.writing_content = content;
|
||||
}
|
||||
let ordinal = self.next_payload_ordinal;
|
||||
let raw_payload_id = format!("raw_payload:{ordinal}");
|
||||
let shard = self.shards.append(&raw_payload_id, body)?;
|
||||
// Layout is metadata. Applying the individual body cap to it would hide surviving siblings.
|
||||
let retained = if parts.is_empty() {
|
||||
body.len().min(self.limits.max_payload_bytes)
|
||||
} else {
|
||||
body.len()
|
||||
};
|
||||
// Leave room for lifecycle events and their explicit omitted-body references.
|
||||
let reserve = (self.limits.max_bytes / 8).min(1 << 20);
|
||||
let shard_limit = self
|
||||
.limits
|
||||
.max_bytes
|
||||
.saturating_sub(self.metadata_bytes + reserve + if content { reserve } else { 0 });
|
||||
let shard =
|
||||
self.shards
|
||||
.append_with_limit(&raw_payload_id, &body[..retained], shard_limit)?;
|
||||
let capture = if shard.is_none() {
|
||||
PayloadCapture::Omitted { original_bytes }
|
||||
} else if retained != body.len() {
|
||||
PayloadCapture::Truncated {
|
||||
original_bytes,
|
||||
retained_bytes: retained,
|
||||
}
|
||||
} else {
|
||||
PayloadCapture::Complete
|
||||
};
|
||||
self.next_payload_ordinal += 1;
|
||||
Ok(RawPayloadRef {
|
||||
raw_payload_id,
|
||||
kind,
|
||||
shard: Some(shard),
|
||||
capture: crate::PayloadCapture::Complete,
|
||||
parts: Vec::new(),
|
||||
shard,
|
||||
capture,
|
||||
parts,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -104,8 +205,16 @@ impl BundleWriter {
|
||||
codex_turn_id: context.codex_turn_id,
|
||||
payload,
|
||||
};
|
||||
serde_json::to_writer(&mut self.events, &event)?;
|
||||
self.events.write_all(b"\n")?;
|
||||
let mut bytes = serde_json::to_vec(&event)?;
|
||||
bytes.push(b'\n');
|
||||
if self.metadata_bytes + self.shards.charged_bytes() + bytes.len() as u64
|
||||
> self.limits.max_bytes
|
||||
{
|
||||
self.event_log_full = true;
|
||||
bail!("trace event log reached its byte limit");
|
||||
}
|
||||
self.events.write_all(&bytes)?;
|
||||
self.metadata_bytes += bytes.len() as u64;
|
||||
self.events.flush()?;
|
||||
self.next_seq += 1;
|
||||
Ok(event)
|
||||
@@ -120,7 +229,22 @@ impl TraceWriter {
|
||||
rollout_id: String,
|
||||
root_thread_id: AgentThreadId,
|
||||
) -> Result<Self> {
|
||||
let bundle_dir = bundle_dir.as_ref();
|
||||
Self::create_with_limits(
|
||||
bundle_dir.as_ref(),
|
||||
trace_id,
|
||||
rollout_id,
|
||||
root_thread_id,
|
||||
TraceCaptureLimits::default(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn create_with_limits(
|
||||
bundle_dir: &Path,
|
||||
trace_id: String,
|
||||
rollout_id: String,
|
||||
root_thread_id: AgentThreadId,
|
||||
limits: TraceCaptureLimits,
|
||||
) -> Result<Self> {
|
||||
let payloads_dir = bundle_dir.join(PAYLOADS_DIR_NAME);
|
||||
std::fs::create_dir_all(&payloads_dir)
|
||||
.with_context(|| format!("create trace payload dir {}", payloads_dir.display()))?;
|
||||
@@ -128,11 +252,17 @@ impl TraceWriter {
|
||||
let started_at_unix_ms = unix_time_ms();
|
||||
let manifest =
|
||||
TraceBundleManifest::new(trace_id, rollout_id, root_thread_id, started_at_unix_ms);
|
||||
let manifest_bytes = serde_json::to_vec_pretty(&manifest)?;
|
||||
let metadata_bytes = manifest_bytes.len() as u64;
|
||||
anyhow::ensure!(
|
||||
metadata_bytes <= limits.max_bytes,
|
||||
"manifest exceeds trace byte limit"
|
||||
);
|
||||
let event_log_path = bundle_dir.join(RAW_EVENT_LOG_FILE_NAME);
|
||||
let event_log = File::create_new(&event_log_path)
|
||||
.with_context(|| format!("open trace event log {}", event_log_path.display()))?;
|
||||
let manifest_path = bundle_dir.join(MANIFEST_FILE_NAME);
|
||||
std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?)
|
||||
std::fs::write(&manifest_path, &manifest_bytes)
|
||||
.with_context(|| format!("write trace manifest {}", manifest_path.display()))?;
|
||||
|
||||
let failure = Arc::new(OnceLock::new());
|
||||
@@ -140,6 +270,11 @@ impl TraceWriter {
|
||||
rollout_id: manifest.rollout_id,
|
||||
events: BufWriter::new(event_log),
|
||||
shards: ShardWriter::new(payloads_dir),
|
||||
limits,
|
||||
metadata_bytes,
|
||||
event_log_full: false,
|
||||
items: HashMap::new(),
|
||||
writing_content: false,
|
||||
next_seq: 1,
|
||||
next_payload_ordinal: 1,
|
||||
failure: Arc::clone(&failure),
|
||||
@@ -181,7 +316,7 @@ impl TraceWriter {
|
||||
result.recv().context("rollout trace worker stopped")?
|
||||
}
|
||||
|
||||
/// Publishes a complete JSON body and returns its location in the bundle.
|
||||
/// Records a JSON body and returns its capture status and storage reference.
|
||||
pub fn write_json_payload(
|
||||
&self,
|
||||
kind: RawPayloadKind,
|
||||
@@ -448,3 +583,7 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "capture_tests.rs"]
|
||||
pub(crate) mod capture_tests;
|
||||
|
||||
@@ -356,6 +356,7 @@ async fn new_config(
|
||||
log_dir: codex_home.join("log").to_path_buf(),
|
||||
codex_home,
|
||||
history: History::default(),
|
||||
rollout_trace_limits: Default::default(),
|
||||
ephemeral: true,
|
||||
allow_persistent_dynamic_tool_updates: false,
|
||||
extra_config: None,
|
||||
|
||||
Reference in New Issue
Block a user