diff --git a/codex-rs/cli/tests/rollout_trace.rs b/codex-rs/cli/tests/rollout_trace.rs index 78d5802965..9b290a2b2c 100644 --- a/codex-rs/cli/tests/rollout_trace.rs +++ b/codex-rs/cli/tests/rollout_trace.rs @@ -34,7 +34,9 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> { let codex = codex_utils_cargo_bin::cargo_bin("codex")?; let home = TempDir::new()?; let server = create_mock_responses_server_sequence(Vec::new()).await; - MockResponsesConfig::new(&server.uri()).write(home.path())?; + MockResponsesConfig::new(&server.uri()) + .with_extra_config("[rollout_trace]\nmax_payload_bytes = 64") + .write(home.path())?; let trace_root = home.path().join("traces"); std::fs::write(&trace_root, "file")?; let mut app = TestAppServer::builder() @@ -61,16 +63,39 @@ async fn enabled_recording_cannot_silently_fail_to_start() -> Result<()> { std::fs::remove_file(&trace_root)?; std::fs::create_dir(&trace_root)?; app.start_thread(ThreadStartParams::default()).await?; - assert!(app.shutdown_gracefully().await?.success()); let mut bundles = std::fs::read_dir(trace_root)?; let bundle = bundles.next().unwrap()?.path(); assert!(bundles.next().is_none()); - assert_eq!( - reduce_with_recording_binary(&codex, home.path(), &bundle) - .await? - .recording_status, - codex_rollout_trace::RecordingStatus::Complete + + // A separate root with a budget too small for its manifest must fail at startup. + let request_id = app + .send_thread_start_request_with_auto_env(ThreadStartParams { + config: Some([("rollout_trace".into(), json!({"max_bytes": 1}))].into()), + ..Default::default() + }) + .await?; + let error = app + .read_stream_until_error_message(RequestId::Integer(request_id)) + .await?; + assert!( + error + .error + .message + .contains("manifest exceeds trace byte limit") ); + assert!(app.shutdown_gracefully().await?.success()); + let trace = reduce_with_recording_binary(&codex, home.path(), &bundle).await?; + assert_eq!( + trace.recording_status, + codex_rollout_trace::RecordingStatus::ClosedWithPartialPayloads + ); + assert!(trace.raw_payloads.values().any(|p| matches!( + p.capture, + codex_rollout_trace::PayloadCapture::Truncated { + retained_bytes: 64, + .. + } + ))); Ok(()) } diff --git a/codex-rs/config/src/config_toml.rs b/codex-rs/config/src/config_toml.rs index 7b3c753380..c485a2e614 100644 --- a/codex-rs/config/src/config_toml.rs +++ b/codex-rs/config/src/config_toml.rs @@ -161,6 +161,16 @@ pub struct FeatureToggleToml { pub enabled: Option, } +/// Storage budgets for one native rollout trace, shared by the root and its children. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq, JsonSchema)] +#[schemars(deny_unknown_fields)] +pub struct RolloutTraceToml { + /// Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB. + pub max_payload_bytes: Option, + /// Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB. + pub max_bytes: Option, +} + /// Base config deserialized from ~/.codex/config.toml. #[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq, JsonSchema)] #[schemars(deny_unknown_fields)] @@ -371,6 +381,9 @@ pub struct ConfigToml { #[serde(default = "default_history")] pub history: Option, + /// Capture limits for local rollout traces when recording is enabled. + pub rollout_trace: Option, + /// Directory where Codex stores the SQLite state DB. /// Defaults to `$CODEX_SQLITE_HOME` when set. Otherwise uses `$CODEX_HOME`. pub sqlite_home: Option, diff --git a/codex-rs/core/config.schema.json b/codex-rs/core/config.schema.json index ef17655461..b5930b9d94 100644 --- a/codex-rs/core/config.schema.json +++ b/codex-rs/core/config.schema.json @@ -4860,6 +4860,25 @@ }, "type": "object" }, + "RolloutTraceToml": { + "additionalProperties": false, + "description": "Storage budgets for one native rollout trace, shared by the root and its children.", + "properties": { + "max_bytes": { + "description": "Maximum bytes for the manifest, event log, and compressed payload shards. Defaults to 256 MiB.", + "format": "uint64", + "minimum": 0.0, + "type": "integer" + }, + "max_payload_bytes": { + "description": "Maximum uncompressed bytes retained per new body or separated value. Defaults to 64 MiB.", + "format": "uint", + "minimum": 0.0, + "type": "integer" + } + }, + "type": "object" + }, "RuntimeAgentInstancesConfigToml": { "additionalProperties": false, "properties": { @@ -9131,6 +9150,14 @@ "description": "Review model override used by the `/review` feature.", "type": "string" }, + "rollout_trace": { + "allOf": [ + { + "$ref": "#/definitions/RolloutTraceToml" + } + ], + "description": "Capture limits for local rollout traces when recording is enabled." + }, "sandbox_mode": { "allOf": [ { diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index 28efe05e3f..771b60acc5 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -73,6 +73,7 @@ use codex_protocol_test_support::uuid_v7_item_id_generator_for_test; use codex_rollout_trace::ExecutionStatus; use codex_rollout_trace::InferenceTraceAttempt; use codex_rollout_trace::InferenceTraceContext; +use codex_rollout_trace::RawPayloadKind; use codex_rollout_trace::RawTraceEventPayload; use codex_rollout_trace::RolloutTrace; use codex_rollout_trace::TraceWriter; @@ -1510,6 +1511,15 @@ async fn replay_until_cancelled(temp: &TempDir) -> anyhow::Result .next() .expect("inference should be reduced"); if inference.execution.status == ExecutionStatus::Cancelled { + // Cancellation publishes one partial response. Prepared prompts can index shared child records. + assert_eq!( + rollout + .raw_payloads + .values() + .filter(|p| p.kind == RawPayloadKind::InferenceResponse) + .count(), + 1 + ); return Ok(rollout); } tokio::time::sleep(Duration::from_millis(10)).await; @@ -1811,7 +1821,6 @@ async fn dropped_response_stream_traces_cancelled_partial_output() -> anyhow::Re rollout.threads["thread-root"].conversation_item_ids, inference.request_item_ids ); - assert_eq!(rollout.raw_payloads.len(), 3); Ok(()) } @@ -2084,7 +2093,6 @@ async fn dropped_backpressured_response_stream_traces_cancelled_partial_output() assert_eq!(inference.execution.status, ExecutionStatus::Cancelled); assert_eq!(inference.response_item_ids.len(), 1); - assert_eq!(rollout.raw_payloads.len(), 3); Ok(()) } diff --git a/codex-rs/core/src/config/mod.rs b/codex-rs/core/src/config/mod.rs index 2bf6c8cc63..a095bce3a7 100644 --- a/codex-rs/core/src/config/mod.rs +++ b/codex-rs/core/src/config/mod.rs @@ -142,6 +142,7 @@ use codex_protocol::protocol::AskForApproval; use codex_protocol::protocol::MultiAgentVersion; use codex_protocol::protocol::SandboxPolicy; use codex_rmcp_client::McpOAuthRefreshMode; +use codex_rollout_trace::TraceCaptureLimits; use codex_sandboxing::SandboxType; pub use codex_thread_store::ExtraConfig; use codex_utils_absolute_path::AbsolutePathBuf; @@ -1040,6 +1041,9 @@ pub struct Config { /// Settings that govern if and what will be written to `~/.codex/history.jsonl`. pub history: History, + /// Limits shared by all threads recording into one native rollout trace. + pub rollout_trace_limits: TraceCaptureLimits, + /// When true, session is not persisted on disk. Default to `false` pub ephemeral: bool, @@ -4153,6 +4157,12 @@ impl Config { let allow_login_shell = cfg.allow_login_shell.unwrap_or(true); let history = cfg.history.unwrap_or_default(); + let rollout_trace = cfg.rollout_trace.unwrap_or_default(); + let defaults = TraceCaptureLimits::default(); + let rollout_trace_limits = TraceCaptureLimits { + max_payload_bytes: rollout_trace.max_payload_bytes.unwrap_or(defaults.max_payload_bytes), + max_bytes: rollout_trace.max_bytes.unwrap_or(defaults.max_bytes), + }; if multi_agent_v2.max_concurrent_threads_per_session == 0 { return Err(std::io::Error::new( @@ -4762,6 +4772,7 @@ impl Config { application_network_policy: Default::default(), application_auth_route_config: None, history, + rollout_trace_limits, ephemeral: ephemeral.unwrap_or_default(), allow_persistent_dynamic_tool_updates: false, extra_config: None, diff --git a/codex-rs/core/src/session/session.rs b/codex-rs/core/src/session/session.rs index 8be322fadc..14550f7d6e 100644 --- a/codex-rs/core/src/session/session.rs +++ b/codex-rs/core/src/session/session.rs @@ -1904,7 +1904,7 @@ impl Session { parent_rollout_thread_trace.start_child_thread_trace(trace_metadata) } } else { - ThreadTraceContext::start_root(trace_metadata) + ThreadTraceContext::start_root(trace_metadata, config.rollout_trace_limits) }; let rollout_thread_trace = rollout_thread_trace.unwrap_or_else(|err| { startup_error.get_or_insert_with(|| { diff --git a/codex-rs/rollout-trace/Cargo.toml b/codex-rs/rollout-trace/Cargo.toml index 98793fd99b..ebcb676f1c 100644 --- a/codex-rs/rollout-trace/Cargo.toml +++ b/codex-rs/rollout-trace/Cargo.toml @@ -18,7 +18,7 @@ codex-code-mode = { workspace = true } codex-protocol = { workspace = true } http = { workspace = true } serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } +serde_json = { workspace = true, features = ["raw_value"] } sha2 = { workspace = true } tracing = { workspace = true } zstd = { workspace = true } diff --git a/codex-rs/rollout-trace/README.md b/codex-rs/rollout-trace/README.md index 3ac35903dd..e26b199b01 100644 --- a/codex-rs/rollout-trace/README.md +++ b/codex-rs/rollout-trace/README.md @@ -110,9 +110,8 @@ writer. Disabled contexts accept the same calls and record nothing. When tracing is enabled, a thread can start or resume only if its recorder initializes successfully. A later write failure stops recording. Partial -bundles remain available for replay. The reducer sets `recording_status` to `complete` in -`state.json` when it reads the closing record written by the last trace handle, -including when the recorded task was interrupted. +bundles remain available for replay. The reduced `recording_status` is `unclosed` until the closing record arrives. It then becomes +`complete` or `closed_with_partial_payloads` according to the referenced captures. ## Bundle Layout @@ -124,12 +123,26 @@ A trace bundle contains: events, terminal output, compaction data, and protocol snapshots. - `state.json`: reduced trace for consumers, written by `codex debug trace-reduce`. -Schema v3 stores whole JSON bodies in numbered Zstandard shards. Every reference in -`trace.jsonl` carries the payload ID and shard number. The reducer collects these -references in `state.json` under `raw_payloads`, so the same index serves every -payload kind. `PayloadReader::read` resolves a reference to JSON bytes and caches -the two most recently read shards. An oversized shard occupies the cache alone. -Reduce a bundle with the binary that recorded it. +Schema v3 indexes numbered Zstandard shards in `state.json` under `raw_payloads`. +Prepared prompts share exact input items. Inline images have independent references and shards, +so losing an attachment preserves its position, sibling text and tool links. A composite record +stores the literal JSON fragments around its ordered `parts`. Interleaving both reconstructs the +original bytes, including whitespace and escapes. Hashes cover each entire value before truncation. +`PayloadReader::read` returns the original bytes for complete bodies. A truncated body +records its original and retained lengths and can be inspected with `read_prefix`. + +Configure the capture limits in `config.toml`. Both values are in bytes. + +```toml +[rollout_trace] +max_payload_bytes = 67108864 # Default: 64 MiB per new body or separated value +max_bytes = 268435456 # Default: 256 MiB per bundle +``` + +The bundle budget covers the manifest, event log, layouts, and compressed payloads. +Space is reserved for later events carrying explicit omissions. +If even an event cannot fit, the log ends at the preceding complete event. +The optional reduced-state cache is generated separately. Reduce a bundle with its recording binary. For an inference call, `raw_request_payload_id` resolves to the outgoing HTTP body or WebSocket message, including its `previous_response_id` and incremental @@ -150,9 +163,9 @@ and write the closing marker. A recording failure prevents the marker. Each shard is one Zstandard frame (level 3, 8 MiB window, checksum enabled). Each record contains a little-endian 16-bit ID length, a little-endian 64-bit -body length, the UTF-8 ID, then the JSON bytes. The writer flushes a record +retained body length, the UTF-8 ID, then the retained bytes. The writer flushes a record before publishing its reference, and rotates before the next record would -exceed 32 MiB uncompressed. An oversized record occupies one shard. The final +exceed 8 MiB uncompressed. An oversized record occupies one shard. The final frame is sealed before the writer publishes `RecordingCompleted`. A live bundle can be reduced through its published records with the last frame still open. diff --git a/codex-rs/rollout-trace/src/capture_tests.rs b/codex-rs/rollout-trace/src/capture_tests.rs new file mode 100644 index 0000000000..e04f383195 --- /dev/null +++ b/codex-rs/rollout-trace/src/capture_tests.rs @@ -0,0 +1,139 @@ +#![cfg(test)] + +use super::*; +use crate::PayloadReader; +use crate::reducer::test_support::ROOT_THREAD_ID; +use crate::replay_bundle; +use pretty_assertions::assert_eq; +use serde_json::json; +use tempfile::TempDir; + +pub(crate) fn limited_trace(body: usize, bundle: u64) -> Result<(TempDir, TraceWriter)> { + let root = TempDir::new()?; + let writer = TraceWriter::create_with_limits( + root.path(), + "t".into(), + "r".into(), + ROOT_THREAD_ID.into(), + TraceCaptureLimits { + max_payload_bytes: body, + max_bytes: bundle, + }, + )?; + Ok((root, writer)) +} + +#[test] +fn parts_reuse_the_complete_identity_and_round_trip_in_order() -> Result<()> { + let (root, writer) = limited_trace(/*body*/ 80, /*bundle*/ 32 << 10)?; + let prefix = "a".repeat(80); + let oversized = + json!({"input": [format!("{prefix}1"), format!("{prefix}2"), format!("{prefix}1")]}); + let first = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?; + let repeated = writer.write_json_payload(RawPayloadKind::PreparedPrompt, &oversized)?; + let items = &first.parts; + assert_eq!(first.parts, repeated.parts); + assert_eq!(items[0], items[2]); + assert_ne!(items[0].raw_payload_id, items[1].raw_payload_id); + assert_eq!( + items[0].capture, + PayloadCapture::Truncated { + original_bytes: 83, + retained_bytes: 80 + } + ); + let mut reader = PayloadReader::new(root.path()); + assert!(reader.read(&first).is_err()); + let captured = reader.read_prefix(&items[0])?.to_vec(); + assert_eq!(captured, reader.read_prefix(&items[1])?); + + let body = r#"{"z":true,"input":[{"s":"\u0061"},1.0,{"s":"\u0061"}],"a":[]}"#; + let prompt = writer.write_json_payload( + RawPayloadKind::PreparedPrompt, + &RawValue::from_string(body.to_string())?, + )?; + assert_eq!(reader.read(&prompt)?, body.as_bytes()); + writer.append(RawTraceEventPayload::ProtocolEventObserved { + event_type: "test".into(), + event_payload: first, + })?; + drop(writer); + let state = replay_bundle(root.path())?; + assert_eq!( + state.recording_status, + crate::RecordingStatus::ClosedWithPartialPayloads + ); + assert_eq!(state.raw_payloads.len(), 3); + Ok(()) +} + +#[test] +fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()> { + let limit = 4096; + let (root, writer) = limited_trace(/*body*/ 1 << 20, /*bundle*/ limit)?; + let compressible = + writer.write_json_payload(RawPayloadKind::ProtocolEvent, &"x".repeat(8192))?; + assert!(compressible.is_complete()); + let noisy: String = (0u32..256) + .map(|i| format!("{:x}", Sha256::digest(i.to_le_bytes()))) + .collect(); + let omitted = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &noisy)?; + assert_eq!( + (&omitted.capture, omitted.shard), + ( + &PayloadCapture::Omitted { + original_bytes: noisy.len() + 2 + }, + None + ) + ); + crate::reducer::test_support::start_thread(&writer, ROOT_THREAD_ID, "/root")?; + writer.append_with_context( + RawTraceEventContext::turn(ROOT_THREAD_ID, "turn"), + RawTraceEventPayload::HistoryItemAccepted { + item_id: "missing-receipt".into(), + agent_message_payload: Some(omitted.clone()), + }, + )?; + let later = writer.write_json_payload(RawPayloadKind::ToolResult, &json!({"ok": true}))?; + for event_payload in [compressible, omitted, later.clone()] { + writer.append(RawTraceEventPayload::ProtocolEventObserved { + event_type: "test".into(), + event_payload, + })?; + } + drop(writer); + let state = replay_bundle(root.path())?; + assert_eq!( + state.recording_status, + crate::RecordingStatus::ClosedWithPartialPayloads + ); + assert_eq!( + PayloadReader::new(root.path()).read_json(&later)?, + json!({"ok": true}) + ); + let mut bytes = std::fs::metadata(root.path().join("manifest.json"))?.len() + + std::fs::metadata(root.path().join("trace.jsonl"))?.len(); + for entry in std::fs::read_dir(root.path().join(PAYLOADS_DIR_NAME))? { + bytes += entry?.metadata()?.len(); + } + assert!(bytes <= limit); + Ok(()) +} + +#[test] +fn exhausting_event_space_leaves_a_readable_prefix() -> Result<()> { + let (root, writer) = limited_trace(/*body*/ 1024, /*bundle*/ 1024)?; + let event_payload = writer.write_json_payload(RawPayloadKind::ProtocolEvent, &true)?; + let record = |name: String| RawTraceEventPayload::ProtocolEventObserved { + event_type: name, + event_payload: event_payload.clone(), + }; + writer.append(record("small".into()))?; + assert!(writer.append(record("x".repeat(1024))).is_err()); + drop(writer); + let state = replay_bundle(root.path())?; + assert_eq!(state.recording_status, crate::RecordingStatus::Unclosed); + assert_eq!(state.raw_payloads.len(), 1); + Ok(()) +} diff --git a/codex-rs/rollout-trace/src/inference.rs b/codex-rs/rollout-trace/src/inference.rs index 29371b7e3d..f1f5f00c2a 100644 --- a/codex-rs/rollout-trace/src/inference.rs +++ b/codex-rs/rollout-trace/src/inference.rs @@ -487,7 +487,9 @@ mod tests { assert_eq!(inference.codex_turn_id, "turn-1"); assert_eq!(inference.execution.status, ExecutionStatus::Completed); assert_eq!(inference.upstream_request_id, Some("req-1".to_string())); - assert_eq!(rollout.raw_payloads.len(), 3); + let prepared = &rollout.raw_payloads[&inference.prepared_prompt_payload_id]; + assert_eq!(prepared.parts.len(), 1); + assert_eq!(rollout.raw_payloads.len(), 4); Ok(()) } diff --git a/codex-rs/rollout-trace/src/lib.rs b/codex-rs/rollout-trace/src/lib.rs index eb2c6e5335..7b427f1299 100644 --- a/codex-rs/rollout-trace/src/lib.rs +++ b/codex-rs/rollout-trace/src/lib.rs @@ -79,5 +79,6 @@ pub use tool_dispatch::ToolDispatchRequester; pub use tool_dispatch::ToolDispatchResult; /// No-op-capable handle for recording one resolved tool dispatch. pub use tool_dispatch::ToolDispatchTraceContext; +pub use writer::TraceCaptureLimits; /// Append-only writer used by hot-path Codex instrumentation. pub use writer::TraceWriter; diff --git a/codex-rs/rollout-trace/src/payload.rs b/codex-rs/rollout-trace/src/payload.rs index 3b883fcbf4..b6d5d99075 100644 --- a/codex-rs/rollout-trace/src/payload.rs +++ b/codex-rs/rollout-trace/src/payload.rs @@ -93,3 +93,79 @@ pub enum RawPayloadKind { /// Runtime notification payload carried when a child agent reports back to its parent. AgentResult, } + +/// Select whole JSON values whose bytes can be stored independently. Only model-facing +/// payloads use this layout. Other consumers can continue reading their single raw record. +pub(crate) fn capture_parts( + kind: RawPayloadKind, + body: &[u8], + limit: usize, +) -> anyhow::Result, RawPayloadKind)>> { + use serde_json::value::RawValue; + use std::collections::BTreeMap; + + if !matches!( + kind, + RawPayloadKind::PreparedPrompt + | RawPayloadKind::ModelItem + | RawPayloadKind::InferenceRequest + | RawPayloadKind::InferenceResponse + | RawPayloadKind::WarmupContext + | RawPayloadKind::CompactionCheckpoint + ) { + return Ok(Vec::new()); + } + let text = std::str::from_utf8(body)?; + if !matches!(text.trim_start().as_bytes().first(), Some(b'{' | b'[')) + || kind != RawPayloadKind::PreparedPrompt + && text.len() <= limit + && !text.contains("data:image/") + { + return Ok(Vec::new()); + } + let mut pending = vec![(String::new(), serde_json::from_str::<&RawValue>(text)?)]; + let mut ranges = Vec::new(); + while let Some((field, raw)) = pending.pop() { + let value = raw.get(); + let mut select = |value: &str, kind| { + let start = value.as_ptr() as usize - text.as_ptr() as usize; + ranges.push((start..start + value.len(), kind)); + }; + match value.as_bytes()[0] { + b'[' if (kind == RawPayloadKind::PreparedPrompt && field == "input") + || (value.len() > limit + && matches!( + field.as_str(), + "input" | "output_items" | "input_history" | "replacement_history" + )) => + { + for item in serde_json::from_str::>(value)? { + select(item.get(), RawPayloadKind::ModelItem); + } + } + b'"' if (field == "image_url" && value.starts_with("\"data:image/")) + || (field == "text" && value.len() > limit) => + { + select(value, RawPayloadKind::ContentValue); + } + b'{' | b'[' + if value.len() > limit + || value.contains("data:image/") + || (kind == RawPayloadKind::PreparedPrompt && field.is_empty()) => + { + if value.starts_with('{') { + pending.extend(serde_json::from_str::>(value)?); + } else { + pending.extend( + serde_json::from_str::>(value)? + .into_iter() + .map(|item| ("[]".into(), item)), + ); + } + } + _ => {} + } + } + ranges.sort_unstable_by_key(|(range, _)| range.start); + Ok(ranges) +} diff --git a/codex-rs/rollout-trace/src/reducer/conversation.rs b/codex-rs/rollout-trace/src/reducer/conversation.rs index beb8546e64..1fe88f06ef 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation.rs @@ -74,7 +74,10 @@ impl TraceReducer { codex_turn_id: &str, request_payload: &RawPayloadRef, ) -> Result> { - let payload = self.payload_reader.read_json(request_payload)?; + if !request_payload.capture.is_complete() { + return Ok(Vec::new()); + } + let payload = self.payload_reader.observed_json(request_payload)?; let Some(input) = payload.get("input") else { bail!( "inference request payload {} did not contain input", @@ -117,9 +120,12 @@ impl TraceReducer { .map(|inference| { let mut ids = inference.request_item_ids.clone(); ids.extend(inference.response_item_ids.clone()); - ids + let incomplete = std::iter::once(&inference.raw_request_payload_id) + .chain(inference.raw_response_payload_id.as_ref()) + .any(|id| !self.rollout.raw_payloads[id].is_complete()); + (ids, incomplete) }); - let Some(mut item_ids) = previous_items else { + let Some((mut item_ids, incomplete)) = previous_items else { bail!( "incremental inference request {inference_call_id} referenced unknown previous_response_id {previous_response_id}" ); @@ -133,7 +139,9 @@ impl TraceReducer { produced_by: Vec::new(), start_index: item_ids.len(), mode: ReconcileMode::AppendOnly, - snapshot_override: None, + // After a capture gap, the number of known prefix items is not + // a position in the thread's last fully observed snapshot. + snapshot_override: incomplete.then_some(item_ids.as_slice()), }, )?; item_ids.extend(delta_item_ids); @@ -170,7 +178,10 @@ impl TraceReducer { inference_call_id: &InferenceCallId, response_payload: &RawPayloadRef, ) -> Result> { - let payload = self.payload_reader.read_json(response_payload)?; + if !response_payload.capture.is_complete() { + return Ok(Vec::new()); + } + let payload = self.payload_reader.observed_json(response_payload)?; let Some(output_items) = payload.get("output_items").and_then(Value::as_array) else { bail!( "inference response payload {} did not contain output_items", @@ -346,14 +357,18 @@ impl TraceReducer { compaction_id: &CompactionId, checkpoint_payload: &RawPayloadRef, ) -> Result { - let payload = self.payload_reader.read_json(checkpoint_payload)?; - let input_history = required_array(&payload, "input_history", checkpoint_payload)?; - let replacement_history = - required_array(&payload, "replacement_history", checkpoint_payload)?; - - let input_items = normalize::normalize_model_items(input_history, checkpoint_payload)?; - let replacement_items = - normalize::normalize_model_items(replacement_history, checkpoint_payload)?; + let (input_items, replacement_items) = if checkpoint_payload.capture.is_complete() { + let payload = self.payload_reader.observed_json(checkpoint_payload)?; + let input = required_array(&payload, "input_history", checkpoint_payload)?; + let replacement = required_array(&payload, "replacement_history", checkpoint_payload)?; + ( + normalize::normalize_model_items(input, checkpoint_payload)?, + normalize::normalize_model_items(replacement, checkpoint_payload)?, + ) + } else { + // The install event still establishes a compaction boundary. + (Vec::new(), Vec::new()) + }; let input_candidates = self .thread_conversation_snapshots .get(thread_id) diff --git a/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs b/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs index 92484d8a89..37838a31fb 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs @@ -4,7 +4,7 @@ use anyhow::Context; use anyhow::Result; use anyhow::bail; use codex_protocol::models::AgentMessageInputContent; -use codex_protocol::models::ResponseItem; +use serde::Deserialize; use serde_json::Value; use sha2::Digest; use sha2::Sha256; @@ -40,7 +40,9 @@ pub(super) fn normalize_model_items( ) -> Result> { let mut normalized_items = Vec::new(); for item in items { - if item.get("type").and_then(Value::as_str) == Some("additional_tools") { + if item.get("type").and_then(Value::as_str) == Some("additional_tools") + || unavailable_ref(item, raw_payload).is_some() + { continue; } @@ -208,22 +210,19 @@ fn normalize_agent_message_item( raw_payload: &RawPayloadRef, ) -> Result { let raw_payload_id = &raw_payload.raw_payload_id; - let response_item = - serde_json::from_value::(item.clone()).with_context(|| { - format!("failed to parse agent_message item in payload {raw_payload_id}") - })?; - let ResponseItem::AgentMessage { - author, - recipient, - content, - .. - } = response_item - else { - bail!("item in payload {raw_payload_id} was not an agent_message"); - }; - let parts = content - .into_iter() - .map(|content| match content { + let routing = AgentMessageMetadata::deserialize(item).with_context(|| { + format!("failed to parse agent_message routing in payload {raw_payload_id}") + })?; + let mut parts = Vec::new(); + for part in item["content"] + .as_array() + .context("expected agent_message content array")? + { + if let Some(missing) = unavailable_ref(&part["text"], raw_payload) { + parts.push(payload_ref_part("text", missing)); + continue; + } + parts.push(match AgentMessageInputContent::deserialize(part)? { AgentMessageInputContent::InputText { text } => ConversationPart::Text { text }, AgentMessageInputContent::EncryptedContent { encrypted_content } => { ConversationPart::Encoded { @@ -231,8 +230,8 @@ fn normalize_agent_message_item( value: encrypted_content, } } - }) - .collect::>(); + }); + } if parts.is_empty() { bail!("agent_message item in payload {raw_payload_id} contained no content"); } @@ -242,7 +241,7 @@ fn normalize_agent_message_item( role: ConversationRole::Assistant, channel: Some(ConversationChannel::Analysis), kind: ConversationItemKind::Message, - agent_message: Some(AgentMessageMetadata { author, recipient }), + agent_message: Some(routing), body: ConversationBody { parts }, call_id: None, }) @@ -359,6 +358,10 @@ fn append_reasoning_parts( } }; + if let Some(missing) = unavailable_ref(&content_item["text"], raw_payload) { + parts.push(payload_ref_part(expected_type, missing)); + continue; + } let Some(text) = content_item.get("text").and_then(Value::as_str) else { bail!( "reasoning item in payload {} had {expected_type} entry without string text", @@ -400,6 +403,14 @@ fn channel_from_phase(phase: &str) -> Option { } } +/// Only interpret recorder-produced gaps backed by a partial reference in this payload. +fn unavailable_ref<'a>(value: &Value, root: &'a RawPayloadRef) -> Option<&'a RawPayloadRef> { + let marker = value.as_object().filter(|obj| obj.len() == 1)?; + let id = marker.get("$trace_unavailable")?.as_str()?; + root.references() + .find(|r| r.raw_payload_id == id && !r.capture.is_complete()) +} + fn content_parts( content: Option<&Value>, raw_payload: &RawPayloadRef, @@ -416,6 +427,8 @@ fn content_parts( parts.push(ConversationPart::Text { text: text.to_string(), }); + } else if let Some(missing) = unavailable_ref(&part["text"], raw_payload) { + parts.push(payload_ref_part("text", missing)); } } other => { @@ -427,7 +440,10 @@ fn content_parts( serde_json::to_writer(&mut digest, &canonical)?; parts.push(ConversationPart::PayloadRef { label: other.unwrap_or("content").to_string(), - raw_payload_id: raw_payload.raw_payload_id.clone(), + raw_payload_id: unavailable_ref(&part["image_url"], raw_payload) + .unwrap_or(raw_payload) + .raw_payload_id + .clone(), content_sha256: Some(format!("{:x}", digest.finalize())), }); } diff --git a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs index 4dc95a766c..7db46843b8 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs @@ -15,11 +15,13 @@ use crate::model::ToolCallKind; use crate::model::ToolCallSummary; use crate::payload::RawPayloadKind; use crate::raw_event::RawTraceEventPayload; +use crate::reducer::test_support::ROOT_THREAD_ID; use crate::reducer::test_support::append_inference_completion; use crate::reducer::test_support::append_inference_start; use crate::reducer::test_support::create_started_writer; use crate::reducer::test_support::expect_replay_error; use crate::reducer::test_support::message; +use crate::reducer::test_support::start_thread; use crate::reducer::test_support::start_turn; use crate::reducer::test_support::trace_context; use crate::replay_bundle; @@ -1373,17 +1375,31 @@ fn context_compaction_boundary_repeats_prefix_and_reuses_replacement_items() -> #[test] fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Result<()> { - let temp = TempDir::new()?; - let writer = create_started_writer(&temp)?; + late_tool_start_with_image_limit(/*body_limit*/ 2048) +} + +#[test] +fn missing_image_preserves_message_order_and_late_tool_links() -> anyhow::Result<()> { + late_tool_start_with_image_limit(/*body_limit*/ 512) +} + +fn late_tool_start_with_image_limit(body_limit: usize) -> anyhow::Result<()> { + let (temp, writer) = + crate::writer::capture_tests::limited_trace(body_limit, /*bundle*/ 64 << 10)?; + start_thread(&writer, ROOT_THREAD_ID, "/root")?; start_turn(&writer, "turn-1")?; - let request = writer.write_json_payload( - RawPayloadKind::InferenceRequest, - &json!({ - // Cross the 9/10 item-ID boundary so lexical order differs from history order. - "input": vec![message("user", "run tests"); 7] - }), - )?; + // Cross the 9/10 item-ID boundary; one photo may be missing, but its message + // and every subsequent request, tool and output retain their original positions. + let mut input = vec![message("user", "run tests"); 6]; + input.push( + json!({"type":"message", "id":"photo", "role":"user", "content":[ + {"type":"input_text","text":"before"}, + {"type":"input_image","image_url":format!("data:image/png;base64,{}", "x".repeat(1024))}, + {"type":"input_text","text":"after"}]}), + ); + let request = + writer.write_json_payload(RawPayloadKind::InferenceRequest, &json!({"input":input}))?; append_inference_start(&writer, "inference-1", "turn-1", request)?; let response = writer.write_json_payload( @@ -1456,10 +1472,30 @@ fn late_tool_start_links_model_call_and_output_versions_in_order() -> anyhow::Re }, )?; + drop(writer); let rollout = replay_bundle(temp.path())?; let first_inference = &rollout.inference_calls["inference-1"]; let second_inference = &rollout.inference_calls["inference-2"]; let tool_call = &rollout.tool_calls["tool-1"]; + let photo = &rollout.conversation_items[&first_inference.request_item_ids[6]]; + assert_eq!(photo.responses_item_id.as_deref(), Some("photo")); + let [ + ConversationPart::Text { text: before }, + ConversationPart::PayloadRef { raw_payload_id, .. }, + ConversationPart::Text { text: after }, + ] = photo.body.parts.as_slice() + else { + panic!("expected an image between its two text parts"); + }; + assert_eq!((before.as_str(), after.as_str()), ("before", "after")); + assert_eq!( + rollout.raw_payloads[raw_payload_id].is_complete(), + body_limit >= 1024 + ); + assert_eq!( + second_inference.request_item_ids[..7], + first_inference.request_item_ids + ); let output_item_ids = &second_inference.request_item_ids[8..]; assert_eq!( diff --git a/codex-rs/rollout-trace/src/reducer/mod.rs b/codex-rs/rollout-trace/src/reducer/mod.rs index 7479578b1c..61b07c4813 100644 --- a/codex-rs/rollout-trace/src/reducer/mod.rs +++ b/codex-rs/rollout-trace/src/reducer/mod.rs @@ -352,7 +352,10 @@ impl TraceReducer { .thread_id .context("history acceptance has no thread")?; self.thread_mut(&thread_id)?; - if let Some(payload) = agent_message_payload { + // Keep the acceptance edge even when a capture limit hid the receipt body. + if let Some(payload) = agent_message_payload + && payload.capture.is_complete() + { self.record_agent_receipt( &thread_id, event.codex_turn_id, diff --git a/codex-rs/rollout-trace/src/reducer/thread.rs b/codex-rs/rollout-trace/src/reducer/thread.rs index 93a8121822..57d3fa766b 100644 --- a/codex-rs/rollout-trace/src/reducer/thread.rs +++ b/codex-rs/rollout-trace/src/reducer/thread.rs @@ -42,6 +42,7 @@ impl TraceReducer { let metadata = metadata_payload .as_ref() + .filter(|p| p.capture.is_complete()) .map(|payload| self.thread_started_metadata(payload)) .transpose()?; let spawn = metadata @@ -79,6 +80,12 @@ impl TraceReducer { task_name, agent_role, } + } else if thread_id != self.rollout.root_thread_id + && metadata_payload + .as_ref() + .is_some_and(|p| !p.capture.is_complete()) + { + AgentOrigin::Unknown } else { AgentOrigin::Root }; diff --git a/codex-rs/rollout-trace/src/reducer/tool/agents.rs b/codex-rs/rollout-trace/src/reducer/tool/agents.rs index cc3c397bd4..75e65bc422 100644 --- a/codex-rs/rollout-trace/src/reducer/tool/agents.rs +++ b/codex-rs/rollout-trace/src/reducer/tool/agents.rs @@ -72,6 +72,9 @@ impl TraceReducer { tool_call_id: &str, runtime_payload: &RawPayloadRef, ) -> Result<()> { + if !runtime_payload.capture.is_complete() { + return Ok(()); + } let kind = self .rollout .tool_calls @@ -132,7 +135,7 @@ impl TraceReducer { ) -> Result<()> { let kind = self.rollout.tool_calls[tool_call_id].kind.clone(); // MCP results are opaque evidence; they cannot describe agent interactions. - if matches!(kind, ToolCallKind::Mcp { .. }) { + if !runtime_payload.capture.is_complete() || matches!(kind, ToolCallKind::Mcp { .. }) { return Ok(()); } let runtime_payload_json = self.payload_reader.read_json(runtime_payload)?; @@ -259,7 +262,10 @@ impl TraceReducer { })?; let started_at_unix_ms = tool_call.execution.started_at_unix_ms; let message_author = self.agent_path_for_thread(&tool_call.thread_id)?; - let message_content = self.agent_message_content_from_invocation(tool_call_id)?; + let Some(message_content) = self.agent_message_content_from_invocation(tool_call_id)? + else { + return Ok(()); + }; let carried_raw_payload_ids = self.agent_tool_payload_ids(tool_call_id)?; self.queue_or_resolve_agent_interaction_edge(PendingAgentInteractionEdge { edge_id, @@ -277,7 +283,10 @@ impl TraceReducer { }) } - fn agent_message_content_from_invocation(&mut self, tool_call_id: &str) -> Result { + fn agent_message_content_from_invocation( + &mut self, + tool_call_id: &str, + ) -> Result> { let tool_call = self.rollout.tool_calls.get(tool_call_id).with_context(|| { format!("agent activity referenced unknown tool call {tool_call_id}") })?; @@ -296,6 +305,9 @@ impl TraceReducer { "agent activity tool call {tool_call_id} referenced missing invocation payload {invocation_payload_id}" ) })?; + if !invocation_payload.capture.is_complete() { + return Ok(None); + } let invocation = self.payload_reader.read_json(invocation_payload)?; let arguments = invocation .get("payload") @@ -306,7 +318,7 @@ impl TraceReducer { })?; let args: AgentMessageInvocationArgs = serde_json::from_str(arguments) .with_context(|| format!("parse agent activity tool call {tool_call_id} arguments"))?; - Ok(args.message) + Ok(Some(args.message)) } /// Adds the canonical tool result payload to an already reduced multi-agent edge. diff --git a/codex-rs/rollout-trace/src/reducer/tool/terminal.rs b/codex-rs/rollout-trace/src/reducer/tool/terminal.rs index 1e0fbfda03..b2401e5fe2 100644 --- a/codex-rs/rollout-trace/src/reducer/tool/terminal.rs +++ b/codex-rs/rollout-trace/src/reducer/tool/terminal.rs @@ -50,7 +50,8 @@ impl TraceReducer { return Ok(None); } let operation_kind = TerminalOperationKind::WriteStdin; - let Some(invocation_payload) = invocation_payload else { + let Some(invocation_payload) = invocation_payload.filter(|p| p.capture.is_complete()) + else { // Payload writes are best-effort in the live recorder. If the // canonical invocation is missing, keep the ToolCall but avoid // fabricating a lossy terminal row. @@ -92,6 +93,9 @@ impl TraceReducer { return Ok(None); }; + if !runtime_payload.capture.is_complete() { + return Ok(None); + } let payload = self.payload_reader.read_json(runtime_payload)?; let payload: ExecCommandBeginPayload = serde_json::from_value(payload).with_context(|| { @@ -172,14 +176,10 @@ impl TraceReducer { bail!("terminal end referenced unknown operation {operation_id}"); }; let response = response_payload + .filter(|p| p.capture.is_complete()) .map(|payload| { let value = self.payload_reader.read_json(payload)?; - let response = parse_terminal_response_payload( - value, - &operation_kind, - &payload.raw_payload_id, - )?; - Ok::<_, anyhow::Error>((payload.raw_payload_id.clone(), response)) + parse_terminal_response_payload(value, &operation_kind, &payload.raw_payload_id) }) .transpose()?; @@ -191,8 +191,10 @@ impl TraceReducer { operation.execution.ended_seq = Some(seq); operation.execution.status = status; - if let Some((raw_payload_id, response)) = response { - push_unique(&mut operation.raw_payload_ids, &raw_payload_id); + if let Some(payload) = response_payload { + push_unique(&mut operation.raw_payload_ids, &payload.raw_payload_id); + } + if let Some(response) = response { operation.result = Some(response.result); response.process_id } else { diff --git a/codex-rs/rollout-trace/src/shard.rs b/codex-rs/rollout-trace/src/shard.rs index e47a0e40b1..8989b4b134 100644 --- a/codex-rs/rollout-trace/src/shard.rs +++ b/codex-rs/rollout-trace/src/shard.rs @@ -1,4 +1,4 @@ -//! Whole JSON bodies in ordered Zstandard shards. Each record is a little-endian +//! Retained payload bytes in ordered Zstandard shards. Each record is a little-endian //! u16 ID length and u64 body length, followed by the UTF-8 ID and the body bytes. //! A flush publishes one record without resetting the compression window. @@ -24,8 +24,9 @@ use crate::payload::PayloadCapture; use crate::payload::RawPayloadId; use crate::payload::RawPayloadRef; -const TARGET_BYTES: usize = 32 * 1024 * 1024; -const WINDOW_LOG: u32 = 23; // 8 MiB of match history across payloads. +const WINDOW_LOG: u32 = 23; +// One match window per shard lets a 16 MiB hosted export admit whole media shards. +const TARGET_BYTES: usize = 1 << WINDOW_LOG; const HEADER_BYTES: usize = 10; fn shard_path(directory: &Path, number: u64) -> PathBuf { @@ -37,6 +38,8 @@ pub(crate) struct ShardWriter { active: Option>>, number: u64, bytes: usize, + closed_bytes: u64, + active_bytes: u64, } impl ShardWriter { @@ -46,12 +49,49 @@ impl ShardWriter { active: None, number: 0, bytes: 0, + closed_bytes: 0, + active_bytes: 0, } } + /// Reserve the final empty block and checksum while a flushed frame is still live. + pub(crate) fn charged_bytes(&self) -> u64 { + self.closed_bytes + self.active_bytes + if self.active.is_some() { 7 } else { 0 } + } + + /// Near the budget boundary, trial-compress one independent frame. A body that + /// does not fit then leaves both the published prefix and future small writes intact. + pub(crate) fn append_with_limit( + &mut self, + id: &str, + body: &[u8], + limit: u64, + ) -> Result> { + let size = HEADER_BYTES + id.len() + body.len(); + let bound = zstd::zstd_safe::compress_bound(size) as u64 + 128; + if self.charged_bytes().saturating_add(bound) <= limit { + return self.append(id, body).map(Some); + } + self.finish()?; + if self.closed_bytes >= limit { + return Ok(None); + } + let mut encoder = new_encoder(Vec::new())?; + write_record(&mut encoder, id, body)?; + let frame = encoder.finish()?; + if frame.len() as u64 > limit - self.closed_bytes { + return Ok(None); + } + let number = self.number + 1; + let mut file = File::create_new(shard_path(&self.directory, number))?; + file.write_all(&frame)?; + self.closed_bytes += frame.len() as u64; + self.number = number; + Ok(Some(number)) + } + /// Returns the shard containing this whole record once it is readable. pub(crate) fn append(&mut self, id: &str, body: &[u8]) -> Result { - let id_len = u16::try_from(id.len())?; let size = HEADER_BYTES + id.len() + body.len(); if self.bytes.saturating_add(size) > TARGET_BYTES { self.finish()?; @@ -64,17 +104,12 @@ impl ShardWriter { let path = shard_path(&self.directory, self.number); let file = File::create_new(&path) .with_context(|| format!("create shard {}", path.display()))?; - let mut encoder = Encoder::new(BufWriter::new(file), 3)?; - encoder.window_log(WINDOW_LOG)?; - encoder.include_checksum(true)?; - self.active.insert(encoder) + self.active.insert(new_encoder(BufWriter::new(file))?) } }; - encoder.write_all(&id_len.to_le_bytes())?; - encoder.write_all(&(body.len() as u64).to_le_bytes())?; - encoder.write_all(id.as_bytes())?; - encoder.write_all(body)?; + write_record(encoder, id, body)?; encoder.flush()?; + self.active_bytes = encoder.get_ref().get_ref().metadata()?.len(); self.bytes += size; Ok(self.number) } @@ -82,12 +117,30 @@ impl ShardWriter { /// Flushes the frame footer before the event log can publish RecordingCompleted. pub(crate) fn finish(&mut self) -> Result<()> { if let Some(active) = self.active.take() { - active.finish()?.flush()?; + let mut file = active.finish()?; + file.flush()?; + self.closed_bytes += file.get_ref().metadata()?.len(); + self.active_bytes = 0; } Ok(()) } } +fn new_encoder(sink: W) -> Result> { + let mut encoder = Encoder::new(sink, 3)?; + encoder.window_log(WINDOW_LOG)?; + encoder.include_checksum(true)?; + Ok(encoder) +} + +fn write_record(sink: &mut impl Write, id: &str, body: &[u8]) -> Result<()> { + sink.write_all(&u16::try_from(id.len())?.to_le_bytes())?; + sink.write_all(&(body.len() as u64).to_le_bytes())?; + sink.write_all(id.as_bytes())?; + sink.write_all(body)?; + Ok(()) +} + struct DecodedShard { number: u64, sealed: bool, diff --git a/codex-rs/rollout-trace/src/thread.rs b/codex-rs/rollout-trace/src/thread.rs index 1ed714cb16..067c7c0735 100644 --- a/codex-rs/rollout-trace/src/thread.rs +++ b/codex-rs/rollout-trace/src/thread.rs @@ -33,6 +33,7 @@ use crate::RolloutStatus; use crate::ToolCallId; use crate::ToolDispatchInvocation; use crate::ToolDispatchTraceContext; +use crate::TraceCaptureLimits; use crate::TraceWriter; use crate::protocol_event::codex_turn_trace_event; use crate::protocol_event::tool_runtime_trace_event; @@ -101,11 +102,14 @@ impl ThreadTraceContext { } /// Starts tracing when configured; an enabled recorder must initialize successfully. - pub fn start_root(metadata: ThreadStartedTraceMetadata) -> anyhow::Result { + pub fn start_root( + metadata: ThreadStartedTraceMetadata, + limits: TraceCaptureLimits, + ) -> anyhow::Result { let Some(root) = std::env::var_os(CODEX_ROLLOUT_TRACE_ROOT_ENV) else { return Ok(Self::disabled()); }; - start_root_in_root(&PathBuf::from(root), metadata) + start_root_in_root(&PathBuf::from(root), metadata, limits) } /// Starts a root trace in a known directory. @@ -116,7 +120,7 @@ impl ThreadTraceContext { root: &Path, metadata: ThreadStartedTraceMetadata, ) -> anyhow::Result { - start_root_in_root(root, metadata) + start_root_in_root(root, metadata, TraceCaptureLimits::default()) } /// Starts one thread lifecycle inside an existing rollout bundle. @@ -551,15 +555,17 @@ impl ThreadTraceContext { fn start_root_in_root( root: &Path, metadata: ThreadStartedTraceMetadata, + limits: TraceCaptureLimits, ) -> anyhow::Result { let trace_id = Uuid::new_v4().to_string(); let thread_id = metadata.thread_id.clone(); let bundle_dir = root.join(format!("trace-{trace_id}-{thread_id}")); - let writer = TraceWriter::create( + let writer = TraceWriter::create_with_limits( &bundle_dir, trace_id.clone(), thread_id.clone(), thread_id.clone(), + limits, )?; let writer = Arc::new(writer); diff --git a/codex-rs/rollout-trace/src/writer.rs b/codex-rs/rollout-trace/src/writer.rs index e14585d143..01d2a91c2c 100644 --- a/codex-rs/rollout-trace/src/writer.rs +++ b/codex-rs/rollout-trace/src/writer.rs @@ -3,6 +3,7 @@ //! A single worker publishes live events in order while model and tool work proceeds. //! Shutdown waits for the worker to drain accepted events and seal the bundle. +use std::collections::HashMap; use std::fs::File; use std::io::BufWriter; use std::io::Write; @@ -14,11 +15,17 @@ use std::thread; use std::time::SystemTime; use std::time::UNIX_EPOCH; +use crate::payload::PayloadCapture; +use crate::payload::capture_parts; use anyhow::Context; use anyhow::Result; use anyhow::anyhow; use anyhow::bail; use serde::Serialize; +#[cfg(test)] +use serde_json::value::RawValue; +use sha2::Digest; +use sha2::Sha256; use crate::bundle::MANIFEST_FILE_NAME; use crate::bundle::PAYLOADS_DIR_NAME; @@ -37,6 +44,24 @@ use crate::shard::ShardWriter; const MAX_PENDING_JOBS: usize = 8; type WriteJob = Box; +/// Logical bytes per new body, and physical bytes for all files in one recording. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TraceCaptureLimits { + /// Maximum uncompressed bytes retained from a single body or separated value. + pub max_payload_bytes: usize, + /// Maximum bytes on disk for the manifest, event log, and compressed payload shards. + pub max_bytes: u64, +} + +impl Default for TraceCaptureLimits { + fn default() -> Self { + Self { + max_payload_bytes: 64 << 20, + max_bytes: 256 << 20, + } + } +} + /// Serializes trace writes in submission order. Explicit writes wait until their evidence is readable. #[derive(Debug)] pub struct TraceWriter { @@ -50,6 +75,11 @@ pub(crate) struct BundleWriter { rollout_id: String, events: BufWriter, shards: ShardWriter, + limits: TraceCaptureLimits, + metadata_bytes: u64, + event_log_full: bool, + items: HashMap<(RawPayloadKind, [u8; 32]), RawPayloadRef>, + writing_content: bool, next_seq: u64, next_payload_ordinal: u64, // Keep the first error so explicit reads and shutdown can detect an incomplete recording. @@ -59,7 +89,13 @@ pub(crate) struct BundleWriter { impl BundleWriter { fn record(&mut self, write: impl FnOnce(&mut Self) -> Result) -> Result { check_recording(&self.failure)?; + if self.event_log_full { + bail!("trace event log reached its byte limit"); + } write(self).inspect_err(|error| { + if self.event_log_full { + return; + } let error = format!("{error:#}"); tracing::error!("rollout recording failed: {error}"); let _ = self.failure.set(error); @@ -71,16 +107,81 @@ impl BundleWriter { kind: RawPayloadKind, body: &[u8], ) -> Result { + let parts = capture_parts(kind, body, self.limits.max_payload_bytes)?; + if parts.is_empty() { + return self.store_body(kind, body, body.len(), Vec::new()); + } + // Literal fragments preserve whitespace, key order and escapes without reserving a + // sentinel in user JSON. A missing child occupies exactly one original value slot. + let mut fragments = Vec::with_capacity(parts.len() + 1); + let mut children = Vec::with_capacity(parts.len()); + let mut end = 0; + for (range, child_kind) in parts { + fragments.push(std::str::from_utf8(&body[end..range.start])?); + end = range.end; + let bytes = &body[range]; + let key = (child_kind, Sha256::digest(bytes).into()); + let child = if let Some(child) = self.items.get(&key) { + child.clone() + } else { + let child = self.write_body(child_kind, bytes)?; + self.items.insert(key, child.clone()); + child + }; + children.push(child); + } + fragments.push(std::str::from_utf8(&body[end..])?); + self.store_body(kind, &serde_json::to_vec(&fragments)?, body.len(), children) + } + + fn store_body( + &mut self, + kind: RawPayloadKind, + body: &[u8], + original_bytes: usize, + parts: Vec, + ) -> Result { + // Bridge admits whole shards. Keep attachment bytes out of shards that carry + // message structure, so omitting a media shard cannot hide a tool or its links. + let content = kind == RawPayloadKind::ContentValue; + if self.writing_content != content { + self.shards.finish()?; + self.writing_content = content; + } let ordinal = self.next_payload_ordinal; let raw_payload_id = format!("raw_payload:{ordinal}"); - let shard = self.shards.append(&raw_payload_id, body)?; + // Layout is metadata. Applying the individual body cap to it would hide surviving siblings. + let retained = if parts.is_empty() { + body.len().min(self.limits.max_payload_bytes) + } else { + body.len() + }; + // Leave room for lifecycle events and their explicit omitted-body references. + let reserve = (self.limits.max_bytes / 8).min(1 << 20); + let shard_limit = self + .limits + .max_bytes + .saturating_sub(self.metadata_bytes + reserve + if content { reserve } else { 0 }); + let shard = + self.shards + .append_with_limit(&raw_payload_id, &body[..retained], shard_limit)?; + let capture = if shard.is_none() { + PayloadCapture::Omitted { original_bytes } + } else if retained != body.len() { + PayloadCapture::Truncated { + original_bytes, + retained_bytes: retained, + } + } else { + PayloadCapture::Complete + }; self.next_payload_ordinal += 1; Ok(RawPayloadRef { raw_payload_id, kind, - shard: Some(shard), - capture: crate::PayloadCapture::Complete, - parts: Vec::new(), + shard, + capture, + parts, }) } @@ -104,8 +205,16 @@ impl BundleWriter { codex_turn_id: context.codex_turn_id, payload, }; - serde_json::to_writer(&mut self.events, &event)?; - self.events.write_all(b"\n")?; + let mut bytes = serde_json::to_vec(&event)?; + bytes.push(b'\n'); + if self.metadata_bytes + self.shards.charged_bytes() + bytes.len() as u64 + > self.limits.max_bytes + { + self.event_log_full = true; + bail!("trace event log reached its byte limit"); + } + self.events.write_all(&bytes)?; + self.metadata_bytes += bytes.len() as u64; self.events.flush()?; self.next_seq += 1; Ok(event) @@ -120,7 +229,22 @@ impl TraceWriter { rollout_id: String, root_thread_id: AgentThreadId, ) -> Result { - let bundle_dir = bundle_dir.as_ref(); + Self::create_with_limits( + bundle_dir.as_ref(), + trace_id, + rollout_id, + root_thread_id, + TraceCaptureLimits::default(), + ) + } + + pub(crate) fn create_with_limits( + bundle_dir: &Path, + trace_id: String, + rollout_id: String, + root_thread_id: AgentThreadId, + limits: TraceCaptureLimits, + ) -> Result { let payloads_dir = bundle_dir.join(PAYLOADS_DIR_NAME); std::fs::create_dir_all(&payloads_dir) .with_context(|| format!("create trace payload dir {}", payloads_dir.display()))?; @@ -128,11 +252,17 @@ impl TraceWriter { let started_at_unix_ms = unix_time_ms(); let manifest = TraceBundleManifest::new(trace_id, rollout_id, root_thread_id, started_at_unix_ms); + let manifest_bytes = serde_json::to_vec_pretty(&manifest)?; + let metadata_bytes = manifest_bytes.len() as u64; + anyhow::ensure!( + metadata_bytes <= limits.max_bytes, + "manifest exceeds trace byte limit" + ); let event_log_path = bundle_dir.join(RAW_EVENT_LOG_FILE_NAME); let event_log = File::create_new(&event_log_path) .with_context(|| format!("open trace event log {}", event_log_path.display()))?; let manifest_path = bundle_dir.join(MANIFEST_FILE_NAME); - std::fs::write(&manifest_path, serde_json::to_vec_pretty(&manifest)?) + std::fs::write(&manifest_path, &manifest_bytes) .with_context(|| format!("write trace manifest {}", manifest_path.display()))?; let failure = Arc::new(OnceLock::new()); @@ -140,6 +270,11 @@ impl TraceWriter { rollout_id: manifest.rollout_id, events: BufWriter::new(event_log), shards: ShardWriter::new(payloads_dir), + limits, + metadata_bytes, + event_log_full: false, + items: HashMap::new(), + writing_content: false, next_seq: 1, next_payload_ordinal: 1, failure: Arc::clone(&failure), @@ -181,7 +316,7 @@ impl TraceWriter { result.recv().context("rollout trace worker stopped")? } - /// Publishes a complete JSON body and returns its location in the bundle. + /// Records a JSON body and returns its capture status and storage reference. pub fn write_json_payload( &self, kind: RawPayloadKind, @@ -448,3 +583,7 @@ mod tests { Ok(()) } } + +#[cfg(test)] +#[path = "capture_tests.rs"] +pub(crate) mod capture_tests; diff --git a/codex-rs/thread-manager-sample/src/main.rs b/codex-rs/thread-manager-sample/src/main.rs index a0744c0cb8..afe4fb0236 100644 --- a/codex-rs/thread-manager-sample/src/main.rs +++ b/codex-rs/thread-manager-sample/src/main.rs @@ -356,6 +356,7 @@ async fn new_config( log_dir: codex_home.join("log").to_path_buf(), codex_home, history: History::default(), + rollout_trace_limits: Default::default(), ephemeral: true, allow_persistent_dynamic_tool_updates: false, extra_config: None,