Files
codex/codex-rs/rollout-trace/src/reducer/conversation/normalize.rs
T
cassirer-openai 508a429666 Deduplicate large tool output strings in rollout traces (#62407)
## Why

Repeated tool output, such as the same image returned in several results,
can consume the trace bundle budget. When output bytes are unavailable,
the surrounding result can still carry terminal metadata and conversation
links.

## What changed

- Store JSON string values of at least 64 KiB, measured after encoding,
  separately from their tool result or runtime event. Identical encoded
  values share one record per bundle, while each occurrence retains its
  position and complete captures reconstruct the original JSON bytes.
- Keep separate compression shards open for content and surrounding
  records. Budget for the serialized event and fall back to a single
  capped or omitted payload when the split layout or its references
  exceed the applicable limit.
- Track missing values separately from literal `null` so reduction can
  preserve surviving terminal metadata and keep tool output IDs attached
  to their original items.

## Testing

Add regression tests for shared image and text payloads, byte-for-byte
reconstruction, recording completion under tight bundle limits, and
reduction with missing output.

GitOrigin-RevId: a003197ea998639b6c2ae27f408990c0186406bd
2026-11-19 02:41:02 +00:00

619 lines
22 KiB
Rust

//! Normalization from Responses-shaped JSON items into conversation item data.
use anyhow::Context;
use anyhow::Result;
use anyhow::bail;
use codex_protocol::models::AgentMessageInputContent;
use serde::Deserialize;
use serde_json::Value;
use sha2::Digest;
use sha2::Sha256;
use std::collections::BTreeMap;
use crate::model::AgentMessageMetadata;
use crate::model::ConversationBody;
use crate::model::ConversationChannel;
use crate::model::ConversationItemKind;
use crate::model::ConversationPart;
use crate::model::ConversationRole;
use crate::model::TokenUsage;
use crate::payload::RawPayloadRef;
/// Conversation fields parsed from one Responses item before trace identity.
///
/// IDs and provenance are assigned after positional reconciliation. Keeping the
/// normalized data separate from `ConversationItem` makes reuse vs insertion a
/// single reducer decision instead of something the parser has to know about.
#[derive(Clone)]
pub(super) struct NormalizedConversationItem {
pub(super) responses_item_id: Option<String>,
pub(super) role: ConversationRole,
pub(super) channel: Option<ConversationChannel>,
pub(super) kind: ConversationItemKind,
pub(super) agent_message: Option<AgentMessageMetadata>,
pub(super) body: ConversationBody,
pub(super) structured_output_hash: Option<[u8; 32]>,
pub(super) call_id: Option<String>,
}
impl NormalizedConversationItem {
pub(super) fn is_delivery_or_tool_output(&self) -> bool {
self.agent_message.is_some()
|| matches!(
self.kind,
ConversationItemKind::FunctionCallOutput
| ConversationItemKind::CustomToolCallOutput
)
}
}
pub(super) fn normalize_model_items(
items: &[Value],
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<Vec<NormalizedConversationItem>> {
let mut normalized_items = Vec::new();
for (index, item) in items.iter().enumerate() {
// Tools and reasoning settings configure inference. Their raw payload retains the control.
if matches!(
item.get("type").and_then(Value::as_str),
Some("additional_tools" | "configuration_update")
) || missing[index].is_string()
{
continue;
}
let mut model_visible_item = item.clone();
if let Some(object) = model_visible_item.as_object_mut() {
object.remove("internal_chat_message_metadata_passthrough");
}
normalized_items.push(normalize_model_item(
&mut model_visible_item,
raw_payload,
&missing[index],
)?);
}
Ok(normalized_items)
}
pub(super) fn token_usage_from_value(value: &Value) -> Option<TokenUsage> {
Some(TokenUsage {
input_tokens: u64_field(value, "input_tokens")?,
cached_input_tokens: u64_field(value, "cached_input_tokens")?,
cache_write_input_tokens: u64_field(value, "cache_write_input_tokens").unwrap_or(0),
output_tokens: u64_field(value, "output_tokens")?,
reasoning_output_tokens: u64_field(value, "reasoning_output_tokens")?,
})
}
pub(super) fn normalize_model_item(
item: &mut Value,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<NormalizedConversationItem> {
let Some(item_type) = item.get("type").and_then(Value::as_str) else {
bail!(
"model item in payload {} did not contain a string type",
raw_payload.raw_payload_id
);
};
let mut normalized = match item_type {
"message" => normalize_message_item(item, raw_payload, missing),
"agent_message" => normalize_agent_message_item(item, raw_payload, missing),
"reasoning" => normalize_reasoning_item(item, raw_payload, missing),
"function_call" => Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::FunctionCall,
agent_message: None,
body: raw_text_or_json_body(item.get("arguments"), raw_payload),
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
}),
"function_call_output" => Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Tool,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::FunctionCallOutput,
agent_message: None,
body: tool_output_body(item.get("output"), raw_payload, &missing["output"])?,
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
}),
"custom_tool_call" => Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::CustomToolCall,
agent_message: None,
body: custom_tool_call_body(item, raw_payload),
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
}),
"custom_tool_call_output" => Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Tool,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::CustomToolCallOutput,
agent_message: None,
body: tool_output_body(item.get("output"), raw_payload, &missing["output"])?,
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
}),
"tool_search_call" | "web_search_call" | "image_generation_call" | "local_shell_call" => {
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::FunctionCall,
agent_message: None,
body: json_body(item, raw_payload),
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
})
}
"tool_search_output" | "mcp_tool_call_output" => {
// A preview can truncate a difference or show null for two different missing
// values. Hash the full visible JSON and the missing child IDs at their paths.
let body = json_body(item, raw_payload);
item.sort_all_objects();
let gaps: BTreeMap<_, _> = missing
.as_object()
.into_iter()
.flatten()
.filter(|(key, _)| item.get(key).is_some())
.collect();
let mut hash = Sha256::new();
serde_json::to_writer(&mut hash, &(&*item, gaps))?;
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: Some(hash.finalize().into()),
role: ConversationRole::Tool,
channel: Some(ConversationChannel::Commentary),
kind: ConversationItemKind::FunctionCallOutput,
agent_message: None,
body,
call_id: item
.get("call_id")
.and_then(Value::as_str)
.map(ToString::to_string),
})
}
"compaction" | "compaction_summary" | "context_compaction" => {
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Summary),
kind: ConversationItemKind::Message,
agent_message: None,
body: compaction_body(item, raw_payload)?,
call_id: None,
})
}
_ => bail!(
"unsupported model item type {item_type} in payload {}",
raw_payload.raw_payload_id
),
}?;
normalized.responses_item_id = item.get("id").and_then(Value::as_str).map(str::to_string);
Ok(normalized)
}
fn normalize_message_item(
item: &Value,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<NormalizedConversationItem> {
let Some(role) = item.get("role").and_then(Value::as_str) else {
bail!(
"message item in payload {} did not contain a string role",
raw_payload.raw_payload_id
);
};
let Some(role) = role_from_str(role) else {
bail!(
"unsupported message role {role} in payload {}",
raw_payload.raw_payload_id
);
};
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role,
channel: item
.get("phase")
.and_then(Value::as_str)
.and_then(channel_from_phase),
kind: ConversationItemKind::Message,
agent_message: None,
body: ConversationBody {
parts: content_parts(item.get("content"), raw_payload, &missing["content"])?,
},
call_id: None,
})
}
fn normalize_agent_message_item(
item: &Value,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<NormalizedConversationItem> {
let raw_payload_id = &raw_payload.raw_payload_id;
let routing = AgentMessageMetadata::deserialize(item).with_context(|| {
format!("failed to parse agent_message routing in payload {raw_payload_id}")
})?;
let mut parts = Vec::new();
for (index, part) in item["content"]
.as_array()
.context("expected agent_message content array")?
.iter()
.enumerate()
{
if let Some(missing) = missing["content"][index]["text"].as_str() {
parts.push(payload_ref_part("text", missing));
continue;
}
parts.push(match AgentMessageInputContent::deserialize(part)? {
AgentMessageInputContent::InputText { text } => ConversationPart::Text { text },
AgentMessageInputContent::EncryptedContent { encrypted_content } => {
ConversationPart::Encoded {
label: "encrypted_content".to_string(),
value: encrypted_content,
}
}
});
}
if parts.is_empty() {
bail!("agent_message item in payload {raw_payload_id} contained no content");
}
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Analysis),
kind: ConversationItemKind::Message,
agent_message: Some(routing),
body: ConversationBody { parts },
call_id: None,
})
}
fn normalize_reasoning_item(
item: &Value,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<NormalizedConversationItem> {
let mut parts = Vec::new();
append_reasoning_parts(
item,
"content",
ReasoningPartKind::Content,
raw_payload,
missing,
&mut parts,
)?;
append_reasoning_parts(
item,
"summary",
ReasoningPartKind::Summary,
raw_payload,
missing,
&mut parts,
)?;
if let Some(encrypted_content) = item.get("encrypted_content") {
let encrypted_content = match encrypted_content {
Value::Null => None,
Value::String(encrypted_content) => Some(encrypted_content),
_ => {
bail!(
"reasoning item in payload {} had non-string encrypted_content",
raw_payload.raw_payload_id
);
}
};
if let Some(encrypted_content) = encrypted_content {
parts.push(ConversationPart::Encoded {
label: "encrypted_content".to_string(),
value: encrypted_content.to_string(),
});
}
}
if parts.is_empty() {
bail!(
"reasoning item in payload {} contained no content, summary, or encrypted_content",
raw_payload.raw_payload_id
);
}
Ok(NormalizedConversationItem {
responses_item_id: None,
structured_output_hash: None,
role: ConversationRole::Assistant,
channel: Some(ConversationChannel::Analysis),
kind: ConversationItemKind::Reasoning,
agent_message: None,
body: ConversationBody { parts },
call_id: None,
})
}
#[derive(Clone, Copy)]
enum ReasoningPartKind {
Content,
Summary,
}
fn append_reasoning_parts(
item: &Value,
key: &str,
kind: ReasoningPartKind,
raw_payload: &RawPayloadRef,
missing: &Value,
parts: &mut Vec<ConversationPart>,
) -> Result<()> {
let Some(items) = item.get(key) else {
return Ok(());
};
if matches!((kind, items), (ReasoningPartKind::Content, Value::Null)) {
return Ok(());
}
let Some(items) = items.as_array() else {
bail!(
"reasoning item in payload {} had non-array {key}",
raw_payload.raw_payload_id
);
};
for (index, content_item) in items.iter().enumerate() {
let Some(item_type) = content_item.get("type").and_then(Value::as_str) else {
bail!(
"reasoning item in payload {} had {key} entry without string type",
raw_payload.raw_payload_id
);
};
let expected_type = match kind {
ReasoningPartKind::Content => {
if !matches!(item_type, "reasoning_text" | "text") {
bail!(
"reasoning item in payload {} had unsupported content type {item_type}",
raw_payload.raw_payload_id
);
}
"content"
}
ReasoningPartKind::Summary => {
if item_type != "summary_text" {
bail!(
"reasoning item in payload {} had unsupported summary type {item_type}",
raw_payload.raw_payload_id
);
}
"summary"
}
};
if let Some(missing) = missing[key][index]["text"].as_str() {
parts.push(payload_ref_part(expected_type, missing));
continue;
}
let Some(text) = content_item.get("text").and_then(Value::as_str) else {
bail!(
"reasoning item in payload {} had {expected_type} entry without string text",
raw_payload.raw_payload_id
);
};
match kind {
ReasoningPartKind::Content => parts.push(ConversationPart::Text {
text: text.to_string(),
}),
ReasoningPartKind::Summary => parts.push(ConversationPart::Summary {
text: text.to_string(),
}),
}
}
Ok(())
}
fn role_from_str(role: &str) -> Option<ConversationRole> {
match role {
"system" => Some(ConversationRole::System),
"developer" => Some(ConversationRole::Developer),
"user" => Some(ConversationRole::User),
"assistant" => Some(ConversationRole::Assistant),
"tool" => Some(ConversationRole::Tool),
_ => None,
}
}
fn channel_from_phase(phase: &str) -> Option<ConversationChannel> {
match phase {
"commentary" => Some(ConversationChannel::Commentary),
// Both answer phases are Harmony final-channel text; only the latter
// declares a terminal answer.
"partial_answer" | "final_answer" => Some(ConversationChannel::Final),
"summary" => Some(ConversationChannel::Summary),
_ => None,
}
}
fn content_parts(
content: Option<&Value>,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<Vec<ConversationPart>> {
let Some(content) = content.and_then(Value::as_array) else {
return Ok(vec![payload_ref_part(
"content",
&raw_payload.raw_payload_id,
)]);
};
let mut parts = Vec::new();
for (index, part) in content.iter().enumerate() {
match part.get("type").and_then(Value::as_str) {
Some("input_text" | "output_text" | "text") => {
if let Some(text) = part.get("text").and_then(Value::as_str) {
parts.push(ConversationPart::Text {
text: text.to_string(),
});
} else if let Some(missing) = missing[index]["text"].as_str() {
parts.push(payload_ref_part("text", missing));
}
}
other => {
// Repeated images live in different request payloads. Compare their
// content without copying large image bodies into the reduced graph.
let mut canonical = part.clone();
canonical.sort_all_objects();
let mut digest = Sha256::new();
serde_json::to_writer(&mut digest, &canonical)?;
let missing_image = missing[index]["image_url"].as_str();
if let Some(id) = missing_image {
digest.update([0]); // Include the shared child's identity in a partial observation.
digest.update(id.as_bytes());
}
parts.push(ConversationPart::PayloadRef {
label: other.unwrap_or("content").to_string(),
raw_payload_id: missing_image
.unwrap_or(&raw_payload.raw_payload_id)
.to_owned(),
content_sha256: Some(format!("{:x}", digest.finalize())),
});
}
}
}
Ok(parts)
}
fn custom_tool_call_body(item: &Value, raw_payload: &RawPayloadRef) -> ConversationBody {
let Some(input) = item.get("input").and_then(Value::as_str) else {
return json_body(item, raw_payload);
};
if item.get("name").and_then(Value::as_str) == Some("exec") {
ConversationBody {
parts: vec![ConversationPart::Code {
language: "javascript".to_string(),
source: input.to_string(),
}],
}
} else {
ConversationBody {
parts: vec![ConversationPart::Text {
text: input.to_string(),
}],
}
}
}
fn raw_text_or_json_body(value: Option<&Value>, raw_payload: &RawPayloadRef) -> ConversationBody {
match value {
Some(Value::String(text)) => {
if let Ok(json) = serde_json::from_str::<Value>(text) {
json_body(&json, raw_payload)
} else {
ConversationBody {
parts: vec![ConversationPart::Text { text: text.clone() }],
}
}
}
Some(value) => json_body(value, raw_payload),
None => ConversationBody {
parts: vec![payload_ref_part("payload", &raw_payload.raw_payload_id)],
},
}
}
fn tool_output_body(
output: Option<&Value>,
raw_payload: &RawPayloadRef,
missing: &Value,
) -> Result<ConversationBody> {
Ok(match output {
Some(Value::String(text)) => ConversationBody {
parts: vec![ConversationPart::Text { text: text.clone() }],
},
Some(Value::Array(_)) => ConversationBody {
parts: content_parts(output, raw_payload, missing)?,
},
Some(value) => json_body(value, raw_payload),
None => ConversationBody {
parts: vec![payload_ref_part("tool_output", &raw_payload.raw_payload_id)],
},
})
}
fn compaction_body(item: &Value, raw_payload: &RawPayloadRef) -> Result<ConversationBody> {
let Some(encrypted_content) = item.get("encrypted_content").and_then(Value::as_str) else {
bail!(
"compaction item in payload {} did not contain string encrypted_content",
raw_payload.raw_payload_id
);
};
// `type: "compaction"` is the remote-compaction summary that later re-enters model requests.
// The structural "history was cut here" marker is inserted separately when the checkpoint is
// installed; payload refs are observation-local, so the encoded summary itself is identity.
Ok(ConversationBody {
parts: vec![ConversationPart::Encoded {
label: "encrypted_content".to_string(),
value: encrypted_content.to_string(),
}],
})
}
fn json_body(value: &Value, raw_payload: &RawPayloadRef) -> ConversationBody {
ConversationBody {
parts: vec![ConversationPart::Json {
summary: summarize_json(value),
raw_payload_id: raw_payload.raw_payload_id.clone(),
}],
}
}
fn payload_ref_part(label: &str, id: &str) -> ConversationPart {
ConversationPart::PayloadRef {
label: label.to_string(),
raw_payload_id: id.to_string(),
content_sha256: None,
}
}
fn summarize_json(value: &Value) -> String {
const MAX_JSON_SUMMARY_LEN: usize = 240;
let mut summary =
serde_json::to_string(value).unwrap_or_else(|_| "<unserializable json>".to_string());
if summary.len() > MAX_JSON_SUMMARY_LEN {
summary.truncate(summary.floor_char_boundary(MAX_JSON_SUMMARY_LEN));
summary.push_str("...");
}
summary
}
fn u64_field(value: &Value, field: &str) -> Option<u64> {
value
.get(field)
.and_then(Value::as_i64)
.map(|value| value.max(0) as u64)
}