Files
codex/codex-rs/rollout-trace/src/inference_recording_tests.rs
T
cassirer-openai c826ca1286 Preserve accepted tool outputs and local resets in rollout traces (#61973)
## Why

Accepted tool outputs may never appear in a later inference request, and request preparation can strip their history IDs. Replay needs to retain these outputs and match them to later snapshots, including trimmed results and outputs with identical wire bodies.

## What changed

- Capture accepted function and custom tool outputs, preserving their acceptance order, identity, and tool provenance during replay.
- Record stripped tool output IDs alongside inference requests so replay can restore identity without changing the captured wire payload. Reuse accepted items and adopt their first model-visible snapshot bodies.
- Record compaction checkpoints for local context-window resets using the selected input and frozen installed history, including goal edits retained during installation.
- Exclude `configuration_update` items from normalized conversation history.

## Testing

Add replay coverage for outputs accepted without subsequent inference, trimmed results, identical outputs with distinct IDs, and acceptance before or after a request snapshot. Extend checkpoint coverage to verify that retained goal edits are captured and later edits do not alter the checkpoint.

GitOrigin-RevId: 70f737782f5000812f389a4e01b9f9690e39549f
2026-11-17 05:35:07 +00:00

149 lines
4.5 KiB
Rust

#![cfg(test)]
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::mpsc;
use std::thread;
use pretty_assertions::assert_eq;
use serde::Serialize;
use serde::Serializer;
use tempfile::TempDir;
use crate::InferenceTraceAttempt;
use crate::InferenceTraceContext;
use crate::PayloadReader;
use crate::RawPayloadKind;
use crate::RawTraceEvent;
use crate::RawTraceEventContext;
use crate::RawTraceEventPayload;
use crate::TraceWriter;
fn recording(dir: &TempDir) -> anyhow::Result<(Arc<TraceWriter>, InferenceTraceAttempt)> {
let writer = Arc::new(TraceWriter::create(
dir.path(),
"trace".into(),
"rollout".into(),
"root".into(),
)?);
let attempt = InferenceTraceContext::enabled(
Arc::clone(&writer),
"root".into(),
"turn".into(),
"model".into(),
"provider".into(),
)
.start_attempt();
Ok((writer, attempt))
}
#[test]
fn inference_dispatch_can_finish_before_prepared_prompt_serialization() -> anyhow::Result<()> {
struct Prompt {
caller: thread::ThreadId,
release: Mutex<mpsc::Receiver<()>>,
}
impl Serialize for Prompt {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
assert_ne!(thread::current().id(), self.caller);
self.release.lock().unwrap().recv().unwrap();
serializer.serialize_str("prepared")
}
}
let dir = TempDir::new()?;
let (writer, attempt) = recording(&dir)?;
let (release, gate) = mpsc::channel();
let wire = br#"{ "input": [] }"#;
attempt.record_started(
wire,
Arc::new(Prompt {
caller: thread::current().id(),
release: Mutex::new(gate),
}),
Vec::new(),
|| Ok(None),
);
attempt.record_completed("response", /*upstream_request_id*/ None, &None, &[]);
release.send(())?;
drop(attempt);
drop(writer);
let events = std::fs::read_to_string(dir.path().join("trace.jsonl"))?
.lines()
.map(serde_json::from_str)
.collect::<Result<Vec<RawTraceEvent>, _>>()?;
let [start, end, closed] = &events[..] else {
panic!("shutdown must publish both accepted events before recording closure")
};
assert!(matches!(
closed.payload,
RawTraceEventPayload::RecordingCompleted
));
let RawTraceEventPayload::InferenceStarted {
request_payload,
prepared_prompt_payload,
..
} = &start.payload
else {
panic!("start must precede completion")
};
assert!(matches!(
end.payload,
RawTraceEventPayload::InferenceCompleted { .. }
));
let mut reader = PayloadReader::new(dir.path());
assert_eq!(reader.read(request_payload)?, wire);
assert_eq!(reader.read(prepared_prompt_payload)?, br#""prepared""#);
Ok(())
}
#[test]
fn deferred_serialization_failure_suppresses_the_start_and_closing_marker() -> anyhow::Result<()> {
let dir = TempDir::new()?;
let (writer, attempt) = recording(&dir)?;
// JSON cannot encode an array as an object key.
attempt.record_started(
b"{}",
Arc::new(BTreeMap::from([([1, 2], true)])),
Vec::new(),
|| Ok(None),
);
assert!(writer.flush().is_err());
drop(attempt);
drop(writer);
assert_eq!(std::fs::read_to_string(dir.path().join("trace.jsonl"))?, "");
Ok(())
}
#[test]
fn borrowed_response_encoding_failure_preserves_the_queued_start() -> anyhow::Result<()> {
let dir = TempDir::new()?;
let (writer, attempt) = recording(&dir)?;
attempt.record_started(b"{}", Arc::new("prepared"), Vec::new(), || Ok(None));
// Handoff succeeds even though JSON cannot represent this response's map key.
writer.event_with_payload(
RawTraceEventContext::turn("root", "turn"),
RawPayloadKind::InferenceResponse,
&BTreeMap::from([([1, 2], true)]),
|_| unreachable!("a failed payload has no reference to publish"),
)?;
assert!(
writer
.flush()
.unwrap_err()
.to_string()
.contains("key must be a string")
);
drop(attempt);
drop(writer);
let log = std::fs::read_to_string(dir.path().join("trace.jsonl"))?;
// Exactly one event survived. Parsing the whole log rejects a spurious closing marker.
let event: RawTraceEvent = serde_json::from_str(&log)?;
assert!(matches!(
event.payload,
RawTraceEventPayload::InferenceStarted { .. }
));
Ok(())
}