From c826ca128699b5464724c8757638ae79f51a45f2 Mon Sep 17 00:00:00 2001 From: cassirer-openai Date: Tue, 17 Nov 2026 05:22:55 +0000 Subject: [PATCH] Preserve accepted tool outputs and local resets in rollout traces (#61973) ## Why Accepted tool outputs may never appear in a later inference request, and request preparation can strip their history IDs. Replay needs to retain these outputs and match them to later snapshots, including trimmed results and outputs with identical wire bodies. ## What changed - Capture accepted function and custom tool outputs, preserving their acceptance order, identity, and tool provenance during replay. - Record stripped tool output IDs alongside inference requests so replay can restore identity without changing the captured wire payload. Reuse accepted items and adopt their first model-visible snapshot bodies. - Record compaction checkpoints for local context-window resets using the selected input and frozen installed history, including goal edits retained during installation. - Exclude `configuration_update` items from normalized conversation history. ## Testing Add replay coverage for outputs accepted without subsequent inference, trimmed results, identical outputs with distinct IDs, and acceptance before or after a request snapshot. Extend checkpoint coverage to verify that retained goal edits are captured and later edits do not alter the checkpoint. GitOrigin-RevId: 70f737782f5000812f389a4e01b9f9690e39549f --- codex-rs/core/src/client.rs | 10 ++ codex-rs/core/src/client_tests.rs | 9 +- codex-rs/core/src/compact_token_budget.rs | 9 +- codex-rs/core/src/session/mod.rs | 53 ++++-- codex-rs/core/src/session/tests.rs | 83 ++++++++-- codex-rs/rollout-trace/src/capture_tests.rs | 1 + codex-rs/rollout-trace/src/compaction.rs | 24 ++- codex-rs/rollout-trace/src/inference.rs | 31 +++- .../src/inference_recording_tests.rs | 12 +- codex-rs/rollout-trace/src/raw_event.rs | 12 +- .../src/reducer/acceptance_tests.rs | 151 +++++++++++++++++- .../rollout-trace/src/reducer/conversation.rs | 125 ++++++++++----- .../src/reducer/conversation/normalize.rs | 18 ++- .../src/reducer/conversation_tests.rs | 1 + .../rollout-trace/src/reducer/inference.rs | 2 + .../src/reducer/inference_tests.rs | 1 + codex-rs/rollout-trace/src/reducer/mod.rs | 17 +- .../rollout-trace/src/reducer/test_support.rs | 1 + codex-rs/rollout-trace/src/reducer/tool.rs | 2 +- .../src/reducer/tool/agents_tests.rs | 1 + codex-rs/rollout-trace/src/thread.rs | 28 +++- codex-rs/rollout-trace/src/thread_tests.rs | 1 + codex-rs/rollout-trace/src/writer.rs | 1 + 23 files changed, 498 insertions(+), 95 deletions(-) diff --git a/codex-rs/core/src/client.rs b/codex-rs/core/src/client.rs index 278ab345c8..971aee4e01 100644 --- a/codex-rs/core/src/client.rs +++ b/codex-rs/core/src/client.rs @@ -2100,6 +2100,7 @@ impl ResponsesHttpExecutor<'_> { )?; self.client .validate_required_cyber_access_program(request.access_programs)?; + let history_ids = inference_trace.stripped_output_ids(&request.input); self.client .prepare_response_items_for_request(&mut request.input); let request_session_telemetry = @@ -2144,6 +2145,7 @@ impl ResponsesHttpExecutor<'_> { inference_trace_attempt.record_started( wire.clone(), Arc::clone(prompt), + history_ids, || Ok(None), ); }) @@ -2416,6 +2418,13 @@ impl ModelClientSession { Some(continuation) => (Some(continuation.response_id), Some(continuation.items)), None => (None, None), }; + // The full context captured after a warmup already carries the original IDs. + let history_ids = if previous_response_id_from_untraced_warmup { + Vec::new() + } else { + inference_trace + .stripped_output_ids(incremental_items.as_deref().unwrap_or(&request.input)) + }; let original_item_ids = if let Some(incremental_items) = &mut incremental_items { self.client .prepare_response_items_for_request(incremental_items); @@ -2505,6 +2514,7 @@ impl ModelClientSession { inference_trace_attempt.record_started( wire.clone(), Arc::clone(prompt), + history_ids, move || { #[derive(serde::Serialize)] struct WarmupContext<'a> { diff --git a/codex-rs/core/src/client_tests.rs b/codex-rs/core/src/client_tests.rs index 771b60acc5..36416ad3ad 100644 --- a/codex-rs/core/src/client_tests.rs +++ b/codex-rs/core/src/client_tests.rs @@ -1478,9 +1478,12 @@ fn started_inference_attempt(temp: &TempDir) -> anyhow::Result { - Complete(CompactedHistoryMetadata), + Complete( + CompactedHistoryMetadata, + Option<(&'a CompactionTraceContext, &'a ContextManager)>, + ), Prefix(PrefixCompactionInstallation<'a>), } impl From for CompactedHistoryInstallation<'_> { fn from(metadata: CompactedHistoryMetadata) -> Self { - Self::Complete(metadata) + Self::Complete(metadata, None) } } @@ -4865,9 +4869,11 @@ impl Session { installation: impl Into>, ) -> bool { let installation = installation.into(); - let compaction_model_hash = match &installation { - CompactedHistoryInstallation::Complete(metadata) => &metadata.compaction_model_hash, - CompactedHistoryInstallation::Prefix(prefix) => &prefix.model_info.comp_hash, + let (compaction_model_hash, trace) = match &installation { + CompactedHistoryInstallation::Complete(metadata, trace) => { + (&metadata.compaction_model_hash, *trace) + } + CompactedHistoryInstallation::Prefix(prefix) => (&prefix.model_info.comp_hash, None), }; crate::model_search::SearchOwner::checkpoint( &self.services.thread_extension_data, @@ -4904,9 +4910,9 @@ impl Session { let (compacted_item, world_state_baseline, started_item) = { let mut state = self.state.lock().await; let complete_replacement = - matches!(&installation, CompactedHistoryInstallation::Complete(_)); + matches!(&installation, CompactedHistoryInstallation::Complete(..)); let replacement = match installation { - CompactedHistoryInstallation::Complete(metadata) => { + CompactedHistoryInstallation::Complete(metadata, _) => { let context = state .additional_context .restore_compacted_items(state.history.annotated_items(), &mut items); @@ -5001,6 +5007,17 @@ impl Session { (compacted_item, world_state_baseline, started_item) }; + // Use the replacement captured under the state lock, including goal edits retained + // by the installer. A later edit or token recomputation cannot change this checkpoint. + if let Some((trace, input)) = trace + && let Some(replacement) = &compacted_item.replacement_history + { + trace.record_local_install( + input.raw_items(), + replacement.iter().map(|entry| &entry.item), + ); + } + if let Some((turn_context, item)) = started_item { // The prefix is now known to be valid; announce the install before checkpoint I/O. self.emit_turn_item_started(turn_context, item).await; @@ -5490,6 +5507,7 @@ impl Session { &self, step_context: &StepContext, world_state: Arc, + trace: &CompactionTraceContext, ) { let turn_context = step_context.turn.as_ref(); let history = self.clone_history().await; @@ -5521,15 +5539,18 @@ impl Session { turn_context_item, world_state_snapshot, /*rejected_audio_file_ids*/ Default::default(), - CompactedHistoryMetadata { - input_goal_ids, - message: String::new(), - window_number, - window_ids, - compaction_response_id: None, - compaction_model_hash: None, - reviewer_compaction_hash: None, - }, + CompactedHistoryInstallation::Complete( + CompactedHistoryMetadata { + input_goal_ids, + message: String::new(), + window_number, + window_ids, + compaction_response_id: None, + compaction_model_hash: None, + reviewer_compaction_hash: None, + }, + Some((trace, &history)), + ), ) .await; self.recompute_token_usage(turn_context).await; diff --git a/codex-rs/core/src/session/tests.rs b/codex-rs/core/src/session/tests.rs index fa91c913e6..a6dd9d837e 100644 --- a/codex-rs/core/src/session/tests.rs +++ b/codex-rs/core/src/session/tests.rs @@ -3944,7 +3944,11 @@ async fn start_new_context_window_persists_checkpoint_state(clear_tools: bool) { ); let expected_snapshot = world_state.render_full().0.into_object(); session - .start_new_context_window(&step_context, world_state) + .start_new_context_window( + &step_context, + world_state, + &codex_rollout_trace::CompactionTraceContext::disabled(), + ) .await; let live_history = session.clone_history().await; @@ -6260,8 +6264,25 @@ async fn compaction_persists_resume_metadata_and_companion_records() { session .record_started_turn(&turn_context.sub_id, Some(turn_context.attribution())) .await; - let input_goal_ids = - crate::context::UserGoalUpdate::message_ids(session.clone_history().await.raw_items()); + let input = session.clone_history().await; + let input_goal_ids = crate::context::UserGoalUpdate::message_ids(input.raw_items()); + let bundle = tempfile::tempdir().unwrap(); + let thread = session.thread_id().to_string(); + let writer = codex_rollout_trace::TraceWriter::create( + bundle.path(), + "trace".into(), + "rollout".into(), + thread.clone(), + ) + .unwrap(); + let trace = codex_rollout_trace::CompactionTraceContext::enabled( + Arc::new(writer), + thread, + turn_context.sub_id.clone(), + "reset".into(), + "test".into(), + "test".into(), + ); // The goal edit is accepted after the compaction input was captured. session .record_user_goal_update(crate::context::UserGoalUpdate::Clear) @@ -6281,15 +6302,18 @@ async fn compaction_persists_resume_metadata_and_companion_records() { turn_context_baseline.clone(), world_state.render_full().0, /*rejected_audio_file_ids*/ Default::default(), - CompactedHistoryMetadata { - input_goal_ids, - message: String::new(), - window_number, - window_ids, - compaction_response_id: None, - compaction_model_hash: None, - reviewer_compaction_hash: None, - }, + CompactedHistoryInstallation::Complete( + CompactedHistoryMetadata { + input_goal_ids, + message: String::new(), + window_number, + window_ids, + compaction_response_id: None, + compaction_model_hash: None, + reviewer_compaction_hash: None, + }, + Some((&trace, &input)), + ), ) .await; let live_history = session.clone_history().await.annotated_items().to_vec(); @@ -6312,6 +6336,41 @@ async fn compaction_persists_resume_metadata_and_companion_records() { else { panic!("unexpected compaction records: {compaction_items:#?}"); }; + // A goal edit accepted after installation must not appear in either checkpoint history. + session + .record_user_goal_update(crate::context::UserGoalUpdate::Clear) + .await + .unwrap(); + drop(trace); // Drain the writer before reading the checkpoint shard. + let events = std::fs::read_to_string(bundle.path().join("trace.jsonl")).unwrap(); + let checkpoint_payload = events + .lines() + .find_map(|line| { + let event: codex_rollout_trace::RawTraceEvent = serde_json::from_str(line).unwrap(); + match event.payload { + codex_rollout_trace::RawTraceEventPayload::CompactionInstalled { + checkpoint_payload, + .. + } => Some(checkpoint_payload), + _ => None, + } + }) + .unwrap(); + let checkpoint = codex_rollout_trace::PayloadReader::new(bundle.path()) + .read_json(&checkpoint_payload) + .unwrap(); + let replacement: Vec<_> = live_history.iter().map(|entry| &entry.item).collect(); + assert_eq!( + checkpoint, + json!({ + "input_history": input.raw_items().collect::>(), + "replacement_history": replacement, + }) + ); + assert_ne!( + session.clone_history().await.annotated_items(), + live_history + ); assert_eq!(compacted.replacement_history, Some(live_history)); assert_eq!(compacted.resume_metadata.as_ref(), Some(&expected)); assert_eq!( diff --git a/codex-rs/rollout-trace/src/capture_tests.rs b/codex-rs/rollout-trace/src/capture_tests.rs index e04f383195..cd12f60e4d 100644 --- a/codex-rs/rollout-trace/src/capture_tests.rs +++ b/codex-rs/rollout-trace/src/capture_tests.rs @@ -93,6 +93,7 @@ fn compressed_budget_admits_small_later_events_after_an_omission() -> Result<()> RawTraceEventPayload::HistoryItemAccepted { item_id: "missing-receipt".into(), agent_message_payload: Some(omitted.clone()), + tool_output_payload: None, }, )?; let later = writer.write_json_payload(RawPayloadKind::ToolResult, &json!({"ok": true}))?; diff --git a/codex-rs/rollout-trace/src/compaction.rs b/codex-rs/rollout-trace/src/compaction.rs index f1f13b84ea..9179334657 100644 --- a/codex-rs/rollout-trace/src/compaction.rs +++ b/codex-rs/rollout-trace/src/compaction.rs @@ -80,9 +80,9 @@ struct TracedCompactionCompleted<'a> { /// `input_history` is the live thread history selected for compaction, while /// `replacement_history` is what future prompts may carry after the checkpoint. #[derive(Serialize)] -pub struct CompactionCheckpointTracePayload<'a> { - pub input_history: &'a [ResponseItem], - pub replacement_history: &'a [ResponseItem], +pub struct CompactionCheckpointTracePayload<'a, T = ResponseItem> { + pub input_history: &'a [T], + pub replacement_history: &'a [T], } impl CompactionTraceContext { @@ -168,6 +168,24 @@ impl CompactionTraceContext { /// The checkpoint belongs to the same semantic compaction lifecycle as the /// compact endpoint attempts, so the context reuses its stable compaction ID. pub fn record_installed(&self, checkpoint: &CompactionCheckpointTracePayload<'_>) { + self.write_checkpoint(checkpoint); + } + + /// Capture the selected input and frozen replacement without copying their message bodies. + pub fn record_local_install<'a>( + &self, + input: impl Iterator, + replacement: impl Iterator, + ) { + if self.is_enabled() { + self.write_checkpoint(&CompactionCheckpointTracePayload { + input_history: &input.collect::>(), + replacement_history: &replacement.collect::>(), + }); + } + } + + fn write_checkpoint(&self, checkpoint: &impl Serialize) { let CompactionTraceContextState::Enabled(context) = &self.state else { return; }; diff --git a/codex-rs/rollout-trace/src/inference.rs b/codex-rs/rollout-trace/src/inference.rs index f1f5f00c2a..7f120677dd 100644 --- a/codex-rs/rollout-trace/src/inference.rs +++ b/codex-rs/rollout-trace/src/inference.rs @@ -122,6 +122,26 @@ impl InferenceTraceContext { } } + /// Original history IDs of tool outputs omitted during request preparation. + pub fn stripped_output_ids(&self, input: &[ResponseItem]) -> Vec<(usize, String)> { + if matches!(self.state, InferenceTraceContextState::Disabled) { + return Vec::new(); + } + let mut ids = Vec::new(); + for (index, item) in input.iter().enumerate() { + if matches!( + item, + ResponseItem::FunctionCallOutput { .. } | ResponseItem::CustomToolCallOutput { .. } + ) && let Some(id) = item + .id() + .filter(|id| id.allocation().is_none() && !id.is_prefixed()) + { + ids.push((index, id.to_string())); + } + } + ids + } + /// Starts a new attempt after the concrete provider request has been built. pub fn start_attempt(&self) -> InferenceTraceAttempt { let InferenceTraceContextState::Enabled(context) = &self.state else { @@ -181,6 +201,7 @@ impl InferenceTraceAttempt { &self, outgoing: impl AsRef<[u8]> + Send + 'static, prepared_prompt: Arc

, + stripped_tool_output_ids: Vec<(usize, String)>, warmup_context: impl FnOnce() -> serde_json::Result>> + Send + 'static, ) { let InferenceTraceAttemptState::Enabled(attempt) = &self.state else { @@ -211,6 +232,7 @@ impl InferenceTraceAttempt { model, provider_name, request_payload, + stripped_tool_output_ids, prepared_prompt_payload, warmup_context_payload, }) @@ -469,9 +491,12 @@ mod tests { "content": [{"type": "input_text", "text": "hello"}] }], }); - attempt.record_started(serde_json::to_vec(&request)?, Arc::new(request), || { - Ok(None) - }); + attempt.record_started( + serde_json::to_vec(&request)?, + Arc::new(request), + Vec::new(), + || Ok(None), + ); attempt.record_completed("resp-1", Some("req-1"), &None, &[]); writer.flush()?; diff --git a/codex-rs/rollout-trace/src/inference_recording_tests.rs b/codex-rs/rollout-trace/src/inference_recording_tests.rs index d7924a7364..07cc6a9f62 100644 --- a/codex-rs/rollout-trace/src/inference_recording_tests.rs +++ b/codex-rs/rollout-trace/src/inference_recording_tests.rs @@ -62,6 +62,7 @@ fn inference_dispatch_can_finish_before_prepared_prompt_serialization() -> anyho caller: thread::current().id(), release: Mutex::new(gate), }), + Vec::new(), || Ok(None), ); attempt.record_completed("response", /*upstream_request_id*/ None, &None, &[]); @@ -102,9 +103,12 @@ fn deferred_serialization_failure_suppresses_the_start_and_closing_marker() -> a let dir = TempDir::new()?; let (writer, attempt) = recording(&dir)?; // JSON cannot encode an array as an object key. - attempt.record_started(b"{}", Arc::new(BTreeMap::from([([1, 2], true)])), || { - Ok(None) - }); + attempt.record_started( + b"{}", + Arc::new(BTreeMap::from([([1, 2], true)])), + Vec::new(), + || Ok(None), + ); assert!(writer.flush().is_err()); drop(attempt); drop(writer); @@ -116,7 +120,7 @@ fn deferred_serialization_failure_suppresses_the_start_and_closing_marker() -> a fn borrowed_response_encoding_failure_preserves_the_queued_start() -> anyhow::Result<()> { let dir = TempDir::new()?; let (writer, attempt) = recording(&dir)?; - attempt.record_started(b"{}", Arc::new("prepared"), || Ok(None)); + attempt.record_started(b"{}", Arc::new("prepared"), Vec::new(), || Ok(None)); // Handoff succeeds even though JSON cannot represent this response's map key. writer.event_with_payload( RawTraceEventContext::turn("root", "turn"), diff --git a/codex-rs/rollout-trace/src/raw_event.rs b/codex-rs/rollout-trace/src/raw_event.rs index 7593bc6c1d..14675db4d3 100644 --- a/codex-rs/rollout-trace/src/raw_event.rs +++ b/codex-rs/rollout-trace/src/raw_event.rs @@ -115,6 +115,9 @@ pub enum RawTraceEventPayload { /// A delivery may never appear in a subsequent inference or compaction snapshot. #[serde(default, skip_serializing_if = "Option::is_none")] agent_message_payload: Option, + /// The accepted output has its history ID and can differ from the dispatch result. + #[serde(default, skip_serializing_if = "Option::is_none")] + tool_output_payload: Option, }, InferenceStarted { inference_call_id: InferenceCallId, @@ -129,6 +132,9 @@ pub enum RawTraceEventPayload { /// Full provider-formatted context only when the previous response was an untraced warmup. #[serde(default, skip_serializing_if = "Option::is_none")] warmup_context_payload: Option, + /// History IDs removed from tool outputs on the wire, indexed within this request input. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + stripped_tool_output_ids: Vec<(usize, String)>, }, InferenceCompleted { inference_call_id: InferenceCallId, @@ -295,8 +301,12 @@ impl RawTraceEventPayload { } => Vec::new(), RawTraceEventPayload::HistoryItemAccepted { agent_message_payload, + tool_output_payload, .. - } => agent_message_payload.iter().collect(), + } => agent_message_payload + .iter() + .chain(tool_output_payload) + .collect(), RawTraceEventPayload::ThreadStarted { metadata_payload, .. } => metadata_payload.iter().collect(), diff --git a/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs b/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs index 74c96d6ebf..95285a26c3 100644 --- a/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/acceptance_tests.rs @@ -13,6 +13,7 @@ use super::test_support::start_thread; use super::test_support::start_turn; use super::test_support::start_turn_for_thread; use super::test_support::trace_context; +use crate::ConversationPart; use crate::ProducerRef; use crate::RawPayloadKind; use crate::RawTraceEventPayload; @@ -42,6 +43,7 @@ fn identical_text_from_a_discarded_attempt_does_not_gain_acceptance() -> anyhow: RawTraceEventPayload::HistoryItemAccepted { item_id: "accepted".into(), agent_message_payload: None, + tool_output_payload: None, }, )?; let response = writer.write_json_payload( @@ -135,10 +137,11 @@ fn late_response_keeps_compaction_replacement_distinct() -> anyhow::Result<()> { "first", vec![input.clone()], )?; + let settings = json!({"type": "configuration_update", "reasoning": {"effort": "high"}}); let checkpoint = writer.write_json_payload( RawPayloadKind::CompactionCheckpoint, &json!({ - "input_history": [input, output], "replacement_history": [output] + "input_history": [settings, input, output], "replacement_history": [settings, output] }), )?; writer.append_with_context( @@ -165,6 +168,8 @@ fn late_response_keeps_compaction_replacement_distinct() -> anyhow::Result<()> { trace.inference_calls["second"].request_item_ids, checkpoint.replacement_item_ids ); + assert_eq!(checkpoint.input_item_ids.len(), 2); + assert_eq!(checkpoint.replacement_item_ids.len(), 1); assert_ne!( checkpoint.input_item_ids[1], checkpoint.replacement_item_ids[0] @@ -222,6 +227,7 @@ fn inherited_output_is_child_prefill_without_copying_parent_sampling_provenance( let accepted = RawTraceEventPayload::HistoryItemAccepted { item_id: "msg-parent".into(), agent_message_payload: None, + tool_output_payload: None, }; let context = trace_context("turn-parent"); let first = writer.append_with_context(context.clone(), accepted.clone())?; @@ -268,3 +274,146 @@ fn replay_rejects_a_mismatched_manifest_schema() -> anyhow::Result<()> { ); Ok(()) } + +fn accept_output(writer: &crate::TraceWriter, item: &serde_json::Value) -> anyhow::Result<()> { + let payload = writer.write_json_payload(RawPayloadKind::ProtocolEvent, item)?; + writer.append_with_context( + trace_context("turn-1"), + RawTraceEventPayload::HistoryItemAccepted { + item_id: item["id"].as_str().unwrap().into(), + agent_message_payload: None, + tool_output_payload: Some(payload), + }, + )?; + Ok(()) +} + +#[test] +fn accepted_tool_results_survive_without_inference_and_reuse_their_ids() -> anyhow::Result<()> { + let temp = TempDir::new()?; + let writer = create_started_writer(&temp)?; + start_turn(&writer, "turn-1")?; + let mut inputs = Vec::new(); + for kind in ["function_call_output", "custom_tool_call_output"] { + let mut item = json!({"id": kind, "type": kind, "call_id": kind, "output": "full result"}); + accept_output(&writer, &item)?; + item["output"] = json!("trimmed result"); + inputs.push(item); + } + let accepted = replay_bundle(temp.path())?; + let ids = &accepted.threads["thread-root"].conversation_item_ids; + assert_eq!(ids.len(), inputs.len()); + assert!(accepted.inference_calls.is_empty()); + for id in ids { + assert!(accepted.conversation_items[id].accepted_seq.is_some()); + } + + append_inference_request(&writer, "thread-root", "turn-1", "next", inputs)?; + let seen = replay_bundle(temp.path())?; + assert_eq!(&seen.inference_calls["next"].request_item_ids, ids); + assert_eq!(&seen.threads["thread-root"].conversation_item_ids, ids); + for id in ids { + assert_eq!( + seen.conversation_items[id].accepted_seq, + accepted.conversation_items[id].accepted_seq + ); + for (state, text) in [(&accepted, "full result"), (&seen, "trimmed result")] { + assert_eq!( + state.conversation_items[id].body.parts, + [ConversationPart::Text { text: text.into() }] + ); + } + } + Ok(()) +} + +#[test] +fn injected_outputs_keep_their_request_positions_and_native_ids() -> anyhow::Result<()> { + for accept_first in [true, false] { + let temp = TempDir::new()?; + let writer = create_started_writer(&temp)?; + start_turn(&writer, "turn-1")?; + let output = json!({"type": "function_call_output", "namespace": "codex_app", "name": "named", "output": "same"}); + let ids = ["fco_first", "legacy-1", "legacy-2"]; + // The first output was inherited. All four have the same wire body. + let settings = json!({"type": "configuration_update", "reasoning": {"effort": "high"}}); + let mut inputs = vec![settings, output.clone()]; + let mut stripped_tool_output_ids = vec![(1, "legacy-inherited".into())]; + for (i, id) in ids.iter().enumerate() { + let mut context = message("developer", &format!("context {i}")); + context["id"] = json!(format!("msg_{i}")); + inputs.push(context); + let mut wire = output.clone(); + if i == 0 { + wire["id"] = json!(id); + } else { + stripped_tool_output_ids.push((inputs.len(), (*id).to_owned())); + } + inputs.push(wire); + } + let accept = || { + ids.iter().try_for_each(|id| { + let mut accepted = output.clone(); + accepted["id"] = json!(id); + accepted["output"] = json!("before trimming"); + accept_output(&writer, &accepted) + }) + }; + if accept_first { + accept()?; + } + let request = json!({"input": inputs}); + let payload = writer.write_json_payload(RawPayloadKind::InferenceRequest, &request)?; + writer.append(RawTraceEventPayload::InferenceStarted { + inference_call_id: "sample".into(), + thread_id: "thread-root".into(), + codex_turn_id: "turn-1".into(), + model: "model".into(), + provider_name: "test".into(), + request_payload: payload.clone(), + prepared_prompt_payload: writer + .write_json_payload(RawPayloadKind::PreparedPrompt, &request)?, + warmup_context_payload: None, + stripped_tool_output_ids, + })?; + if !accept_first { + accept()?; + } + let state = replay_bundle(temp.path())?; + let thread = &state.threads["thread-root"].conversation_item_ids; + assert_eq!(thread, &state.inference_calls["sample"].request_item_ids); + let native_ids: Vec<_> = thread + .iter() + .map(|id| state.conversation_items[id].responses_item_id.as_deref()) + .collect(); + assert_eq!( + native_ids, + [ + "legacy-inherited", + "msg_0", + "fco_first", + "msg_1", + "legacy-1", + "msg_2", + "legacy-2" + ] + .map(Some) + ); + assert!(state.conversation_items[&thread[0]].accepted_seq.is_none()); + for id in thread.iter().skip(2).step_by(2) { + let item = &state.conversation_items[id]; + assert!(item.accepted_seq.is_some()); + assert_eq!( + item.body.parts, + [ConversationPart::Text { + text: "same".into() + }] + ); + } + assert_eq!( + crate::PayloadReader::new(temp.path()).read_json(&payload)?, + request + ); + } + Ok(()) +} diff --git a/codex-rs/rollout-trace/src/reducer/conversation.rs b/codex-rs/rollout-trace/src/reducer/conversation.rs index 1fe88f06ef..a296442486 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation.rs @@ -27,7 +27,7 @@ use crate::payload::RawPayloadRef; mod normalize; impl TraceReducer { - pub(super) fn record_agent_receipt( + pub(super) fn record_accepted_item( &mut self, thread_id: &str, turn_id: Option, @@ -35,30 +35,33 @@ impl TraceReducer { wire_id: &str, payload: &RawPayloadRef, ) -> Result<()> { - // A request can be recorded before its history notification. Its item already owns the edge. - if self.rollout.threads[thread_id] - .conversation_item_ids - .iter() - .rev() - .any(|id| { - self.rollout.conversation_items[id] - .responses_item_id - .as_deref() - == Some(wire_id) - }) - { + // A snapshot can arrive first, or the recorder can repeat a notification. + let key = (thread_id.to_owned(), wire_id.to_owned()); + if self.receivable_ids.contains_key(&key) { return Ok(()); } let item = normalize::normalize_model_item(&self.payload_reader.observed_json(payload)?, payload)?; - if item.agent_message.is_none() || item.responses_item_id.as_deref() != Some(wire_id) { - bail!("accepted agent message does not match receipt {wire_id}"); + if !item.is_delivery_or_tool_output() || item.responses_item_id.as_deref() != Some(wire_id) + { + bail!("accepted item payload does not match {wire_id}"); } - let id = self.create_conversation_item(thread_id, turn_id, at_unix_ms, item, Vec::new()); - self.append_thread_conversation_items(thread_id, std::slice::from_ref(&id))?; - self.resolve_pending_agent_edges_for_item(&id)?; - self.receipt_only_items - .insert((thread_id.to_owned(), wire_id.to_owned()), id); + let is_delivery = item.agent_message.is_some(); + let produced_by = if let Some(call_id) = &item.call_id + && let Some(tool_call_id) = self.single_tool_for_model_visible_call(call_id)? + { + vec![ProducerRef::Tool { tool_call_id }] + } else { + Vec::new() + }; + let id = self.create_conversation_item(thread_id, turn_id, at_unix_ms, item, produced_by); + self.thread_mut(thread_id)? + .conversation_item_ids + .push(id.clone()); + if is_delivery { + self.resolve_pending_agent_edges_for_item(&id)?; + } + self.receivable_ids.insert(key, Some(id)); Ok(()) } @@ -73,24 +76,43 @@ impl TraceReducer { thread_id: &str, codex_turn_id: &str, request_payload: &RawPayloadRef, + stripped_tool_output_ids: Vec<(usize, String)>, ) -> Result> { if !request_payload.capture.is_complete() { return Ok(Vec::new()); } - let payload = self.payload_reader.observed_json(request_payload)?; - let Some(input) = payload.get("input") else { + let mut payload = self.payload_reader.observed_json(request_payload)?; + let Some(input) = payload.get_mut("input") else { bail!( "inference request payload {} did not contain input", request_payload.raw_payload_id ); }; - let Some(request_items) = input.as_array() else { + let Some(request_items) = input.as_array_mut() else { bail!( "inference request payload {} had non-array input", request_payload.raw_payload_id ); }; + // The wire payload stays intact on disk. Resolve accepted history by its native ID, + // even for two identical named outputs or a trimmed result in a websocket delta. + for (index, history_id) in stripped_tool_output_ids { + let item = request_items + .get_mut(index) + .context("stripped tool output index outside request")?; + if item.get("id").is_some() + || !matches!( + item["type"].as_str(), + Some("function_call_output" | "custom_tool_call_output") + ) + { + bail!( + "stripped tool output ID does not refer to an ID-less output at input {index}" + ); + } + item["id"] = Value::String(history_id); + } let items = normalize::normalize_model_items(request_items, request_payload)?; let previous_response_id = payload.get("previous_response_id").and_then(Value::as_str); @@ -484,15 +506,15 @@ impl TraceReducer { item: NormalizedConversationItem, produced_by: Vec, ) -> String { - // The first request or checkpoint that sees a receipt supplies the model-visible - // representation (which can be trimmed). Keep the ID already used by the delivery edge. - if item.agent_message.is_some() + // A request or checkpoint can contain a trimmed version of an accepted item. + // Reuse its graph ID and retain the representation actually seen by the model. + if item.is_delivery_or_tool_output() && let Some(wire_id) = &item.responses_item_id - && let Some(id) = self - .receipt_only_items - .remove(&(thread_id.to_owned(), wire_id.clone())) { - return id; + let key = (thread_id.to_owned(), wire_id.clone()); + if let Some(Some(id)) = self.receivable_ids.get(&key) { + return id.clone(); + } } let item_id = self.next_conversation_item_id(); self.rollout.conversation_items.insert( @@ -534,9 +556,20 @@ impl TraceReducer { if item.kind == ConversationItemKind::Reasoning { merge_reasoning_body(&mut item.body, &normalized.body)?; } - // A request or checkpoint may contain a trimmed version of an accepted delivery. - if normalized.agent_message.is_some() && item.body != normalized.body { - item.body.clone_from(&normalized.body); + // Replace the accepted body on its first snapshot only. Later snapshots may reference + // the same content through a different raw payload. + if normalized.is_delivery_or_tool_output() + && let Some(wire_id) = &normalized.responses_item_id + { + let key = (item.thread_id.clone(), wire_id.clone()); + let accepted_id = self.receivable_ids.entry(key).or_default(); + if accepted_id.as_deref() == Some(item_id) { + *accepted_id = None; + item.body.clone_from(&normalized.body); + if normalized.agent_message.is_none() { + self.snapshot_output_anchors.insert(item_id.to_owned()); + } + } } for producer in produced_by { if !item.produced_by.contains(producer) { @@ -551,10 +584,28 @@ impl TraceReducer { thread_id: &str, item_ids: &[String], ) -> Result<()> { - let thread = self.thread_mut(thread_id)?; - for item_id in item_ids { - if !thread.conversation_item_ids.contains(item_id) { - thread.conversation_item_ids.push(item_id.clone()); + let history = &mut self + .rollout + .threads + .get_mut(thread_id) + .context("unknown thread")? + .conversation_item_ids; + let mut new_prefix = history.len(); + for id in item_ids { + if self.snapshot_output_anchors.remove(id) { + // This output already has its acceptance position. Insert any newly exposed + // context immediately before it, preserving deliveries recorded in between. + if new_prefix < history.len() { + let prefix = history.split_off(new_prefix); + let anchor = history + .iter() + .position(|known| known == id) + .context("missing accepted output")?; + history.splice(anchor..anchor, prefix); + } + new_prefix = history.len(); + } else if !history.contains(id) { + history.push(id.clone()); } } Ok(()) diff --git a/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs b/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs index 37838a31fb..47ea7b96e2 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs @@ -34,14 +34,28 @@ pub(super) struct NormalizedConversationItem { pub(super) call_id: Option, } +impl NormalizedConversationItem { + pub(super) fn is_delivery_or_tool_output(&self) -> bool { + self.agent_message.is_some() + || matches!( + self.kind, + ConversationItemKind::FunctionCallOutput + | ConversationItemKind::CustomToolCallOutput + ) + } +} + pub(super) fn normalize_model_items( items: &[Value], raw_payload: &RawPayloadRef, ) -> Result> { let mut normalized_items = Vec::new(); for item in items { - if item.get("type").and_then(Value::as_str) == Some("additional_tools") - || unavailable_ref(item, raw_payload).is_some() + // Tools and reasoning settings configure inference. Their raw payload retains the control. + if matches!( + item.get("type").and_then(Value::as_str), + Some("additional_tools" | "configuration_update") + ) || unavailable_ref(item, raw_payload).is_some() { continue; } diff --git a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs index 7db46843b8..791bfce686 100644 --- a/codex-rs/rollout-trace/src/reducer/conversation_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/conversation_tests.rs @@ -226,6 +226,7 @@ fn response_outputs_require_harness_acceptance() -> anyhow::Result<()> { RawTraceEventPayload::HistoryItemAccepted { item_id: "msg-accepted".into(), agent_message_payload: None, + tool_output_payload: None, }, )?; diff --git a/codex-rs/rollout-trace/src/reducer/inference.rs b/codex-rs/rollout-trace/src/reducer/inference.rs index 55fa426a47..79b2ae6bfc 100644 --- a/codex-rs/rollout-trace/src/reducer/inference.rs +++ b/codex-rs/rollout-trace/src/reducer/inference.rs @@ -30,6 +30,7 @@ pub(super) struct StartedInferenceCall { pub(super) request_payload: RawPayloadRef, pub(super) prepared_prompt_payload: RawPayloadRef, pub(super) warmup_context_payload: Option, + pub(super) stripped_tool_output_ids: Vec<(usize, String)>, } impl TraceReducer { @@ -92,6 +93,7 @@ impl TraceReducer { &thread_id, &codex_turn_id, context, + started.stripped_tool_output_ids, )?; self.thread_mut(&thread_id)?; diff --git a/codex-rs/rollout-trace/src/reducer/inference_tests.rs b/codex-rs/rollout-trace/src/reducer/inference_tests.rs index 930c85b88e..642a826359 100644 --- a/codex-rs/rollout-trace/src/reducer/inference_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/inference_tests.rs @@ -32,6 +32,7 @@ fn cancelled_inference_retains_accepted_partial_output() -> anyhow::Result<()> { RawTraceEventPayload::HistoryItemAccepted { item_id: "msg-partial".into(), agent_message_payload: None, + tool_output_payload: None, }, )?; let partial_response = writer.write_json_payload( diff --git a/codex-rs/rollout-trace/src/reducer/mod.rs b/codex-rs/rollout-trace/src/reducer/mod.rs index 61b07c4813..3572cd82dd 100644 --- a/codex-rs/rollout-trace/src/reducer/mod.rs +++ b/codex-rs/rollout-trace/src/reducer/mod.rs @@ -80,7 +80,8 @@ pub fn replay_bundle(bundle_dir: impl AsRef) -> Result { pending_code_cell_lifecycle_events: BTreeMap::new(), pending_agent_interaction_edges: Vec::new(), first_acceptances: BTreeMap::new(), - receipt_only_items: BTreeMap::new(), + receivable_ids: BTreeMap::new(), + snapshot_output_anchors: BTreeSet::new(), }; let event_log_path = bundle_dir.join(RAW_EVENT_LOG_FILE_NAME); @@ -206,8 +207,11 @@ struct TraceReducer { payload_reader: PayloadReader, /// First (wall time, sequence) by receiving thread and Responses item ID. first_acceptances: BTreeMap>, - /// Delivered items awaiting their first model-visible sighting, keyed by thread and wire ID. - receipt_only_items: BTreeMap<(String, String), String>, + /// A native delivery or tool output has one graph ID until its first snapshot. + /// None means a snapshot already supplied its body (and a later acceptance is a duplicate). + receivable_ids: BTreeMap<(String, String), Option>, + /// Accepted tool outputs first encountered in the batch about to be appended. + snapshot_output_anchors: BTreeSet, next_conversation_item_ordinal: u64, next_terminal_operation_ordinal: u64, /// Latest session for each thread-local runtime handle; each exec replaces it. @@ -347,16 +351,17 @@ impl TraceReducer { RawTraceEventPayload::HistoryItemAccepted { item_id, agent_message_payload, + tool_output_payload, } => { let thread_id = event .thread_id .context("history acceptance has no thread")?; self.thread_mut(&thread_id)?; // Keep the acceptance edge even when a capture limit hid the receipt body. - if let Some(payload) = agent_message_payload + if let Some(payload) = agent_message_payload.or(tool_output_payload) && payload.capture.is_complete() { - self.record_agent_receipt( + self.record_accepted_item( &thread_id, event.codex_turn_id, event.wall_time_unix_ms, @@ -379,6 +384,7 @@ impl TraceReducer { request_payload, prepared_prompt_payload, warmup_context_payload, + stripped_tool_output_ids, } => { self.start_inference_call( event.seq, @@ -392,6 +398,7 @@ impl TraceReducer { request_payload, prepared_prompt_payload, warmup_context_payload, + stripped_tool_output_ids, }, )?; } diff --git a/codex-rs/rollout-trace/src/reducer/test_support.rs b/codex-rs/rollout-trace/src/reducer/test_support.rs index bf938b04d4..a30fcb96f8 100644 --- a/codex-rs/rollout-trace/src/reducer/test_support.rs +++ b/codex-rs/rollout-trace/src/reducer/test_support.rs @@ -130,6 +130,7 @@ pub(crate) fn append_inference_start_for_thread( prepared_prompt_payload: writer .write_json_payload(RawPayloadKind::PreparedPrompt, &json!({"input": []}))?, warmup_context_payload: None, + stripped_tool_output_ids: Vec::new(), inference_call_id: inference_call_id.to_string(), thread_id: thread_id.to_string(), codex_turn_id: codex_turn_id.to_string(), diff --git a/codex-rs/rollout-trace/src/reducer/tool.rs b/codex-rs/rollout-trace/src/reducer/tool.rs index ace015653c..22758a1e53 100644 --- a/codex-rs/rollout-trace/src/reducer/tool.rs +++ b/codex-rs/rollout-trace/src/reducer/tool.rs @@ -426,7 +426,7 @@ impl TraceReducer { Ok(()) } - fn single_tool_for_model_visible_call( + pub(super) fn single_tool_for_model_visible_call( &self, model_visible_call_id: &str, ) -> Result> { diff --git a/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs b/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs index 293a78927d..2be2f69eea 100644 --- a/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs +++ b/codex-rs/rollout-trace/src/reducer/tool/agents_tests.rs @@ -256,6 +256,7 @@ fn sub_agent_started_activity_creates_spawn_edge() -> anyhow::Result<()> { }); let receipt = RawTraceEventPayload::HistoryItemAccepted { item_id: "amsg-delivery".into(), + tool_output_payload: None, agent_message_payload: Some( writer.write_json_payload(RawPayloadKind::ProtocolEvent, &delivery)?, ), diff --git a/codex-rs/rollout-trace/src/thread.rs b/codex-rs/rollout-trace/src/thread.rs index 067c7c0735..bb2b15fd70 100644 --- a/codex-rs/rollout-trace/src/thread.rs +++ b/codex-rs/rollout-trace/src/thread.rs @@ -249,21 +249,37 @@ impl ThreadTraceContext { let Some(item_id) = event.item.id().map(ToString::to_string) else { return; }; - let accepted = move |agent_message_payload| RawTraceEventPayload::HistoryItemAccepted { - item_id, - agent_message_payload, + let accepted = move |agent_message_payload, tool_output_payload| { + RawTraceEventPayload::HistoryItemAccepted { + item_id, + agent_message_payload, + tool_output_payload, + } }; let turn = RawTraceEventContext::turn(&context.thread_id, default_turn_id); - if matches!(event.item, ResponseItem::AgentMessage { .. }) { + let is_delivery = matches!(event.item, ResponseItem::AgentMessage { .. }); + if is_delivery + || matches!( + event.item, + ResponseItem::FunctionCallOutput { .. } + | ResponseItem::CustomToolCallOutput { .. } + ) + { // Publish the receipt and its history event together, in worker order. let _ = context.writer.event_with_payload( turn, RawPayloadKind::ProtocolEvent, &event.item, - move |payload| accepted(Some(payload)), + move |payload| { + if is_delivery { + accepted(Some(payload), None) + } else { + accepted(None, Some(payload)) + } + }, ); } else { - let _ = context.writer.enqueue_event(turn, accepted(None)); + let _ = context.writer.enqueue_event(turn, accepted(None, None)); } return; } diff --git a/codex-rs/rollout-trace/src/thread_tests.rs b/codex-rs/rollout-trace/src/thread_tests.rs index 00d38746f6..2c49339b71 100644 --- a/codex-rs/rollout-trace/src/thread_tests.rs +++ b/codex-rs/rollout-trace/src/thread_tests.rs @@ -267,6 +267,7 @@ fn disabled_thread_context_accepts_trace_calls_without_writing() -> anyhow::Resu inference_attempt.record_started( serde_json::to_vec(&input)?, std::sync::Arc::new(input), + Vec::new(), || Ok(None), ); let token_usage: Option = None; diff --git a/codex-rs/rollout-trace/src/writer.rs b/codex-rs/rollout-trace/src/writer.rs index 01d2a91c2c..0b2ed01ea5 100644 --- a/codex-rs/rollout-trace/src/writer.rs +++ b/codex-rs/rollout-trace/src/writer.rs @@ -476,6 +476,7 @@ mod tests { prepared_prompt_payload: writer .write_json_payload(RawPayloadKind::PreparedPrompt, &json!({"input": []}))?, warmup_context_payload: None, + stripped_tool_output_ids: Vec::new(), inference_call_id: "inference-1".to_string(), thread_id: "thread-root".to_string(), codex_turn_id: "turn-1".to_string(),