Record Guardian classifier input truncation metrics (#40465)

## What changed

- Track truncation and omission across Guardian v2 actions, transcript entries and images, and synchronous review evidence.
- Emit `codex.guardian_v2.classification.truncation` counters and byte histograms for original, retained, and omitted content after successful classifications.
- Tag observations by input component and whether content was truncated or fully omitted.

## Testing

- Extend async scorer coverage for action truncation metrics.
- Verify the app-server Guardian flow records truncated review evidence.

GitOrigin-RevId: 70680897f75716770aa072f866d60bd37e171c1d
This commit is contained in:
felixxia-oai
2026-08-24 19:47:48 +00:00
committed by copyberry
parent b6333bb1bb
commit c2904b9c5a
8 changed files with 304 additions and 46 deletions
@@ -70,6 +70,7 @@ struct MockResponsesState {
allow_luna: Notify,
allow_guardian_review: Notify,
classification_completed: Notify,
truncation_recorded: Notify,
luna_score: f64,
review_outcome: ReviewOutcome,
transcript_content: TranscriptContent,
@@ -356,6 +357,11 @@ async fn guardian_v2_routes_tool_approvals(
if body.contains("codex.guardian_v2.classification") {
state.classification_completed.notify_one();
}
if body.contains("codex.guardian_v2.classification.truncation")
&& body.contains("sync_review_action")
{
state.truncation_recorded.notify_one();
}
},
),
)
@@ -747,6 +753,10 @@ async fn guardian_v2_routes_tool_approvals(
responses_state.allow_luna.notify_one();
}
if matches!(review_outcome, ReviewOutcome::Deny) {
timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?;
}
mcp_server_handle.abort();
responses_server.abort();
Ok(())
@@ -47,12 +47,18 @@ use super::sampler::LunaSamplerConfig;
use super::sampler::LunaSamplerError;
use super::sampler::LunaSamplingRequest;
use super::sampler::MODEL;
use super::truncation::ClassificationTruncations;
struct GuardianAction {
tool_name: ToolName,
payload: ToolPayload,
}
struct RenderedAction {
text: String,
original_bytes: usize,
}
fn should_classify_tool(
tool_name: &ToolName,
payload: &ToolPayload,
@@ -80,7 +86,7 @@ fn should_classify_tool(
}
impl GuardianAction {
fn render(self, max_action_tokens: usize) -> serde_json::Result<String> {
fn render(self, max_action_tokens: usize) -> serde_json::Result<RenderedAction> {
let arguments = match self.payload {
ToolPayload::Function { arguments } => {
serde_json::from_str(&arguments).unwrap_or(serde_json::Value::String(arguments))
@@ -103,13 +109,20 @@ impl GuardianAction {
.for_each(serde_json::Value::sort_all_objects);
let max_action_bytes = TruncationPolicy::Tokens(max_action_tokens).byte_budget();
let rendered = serde_json::to_string_pretty(&action)?;
let original_bytes = rendered.len();
if rendered.len().saturating_add(1) <= max_action_bytes {
return Ok(rendered);
return Ok(RenderedAction {
text: rendered,
original_bytes,
});
}
if let Some(rendered) = fit_action_to_budget(&action, max_action_bytes, max_action_tokens)?
{
return Ok(rendered);
return Ok(RenderedAction {
text: rendered,
original_bytes,
});
}
let mut omission_key = "_guardian_omitted_fields".to_owned();
@@ -148,10 +161,15 @@ impl GuardianAction {
}
retained.sort_keys();
fit_action_to_budget(&retained, max_action_bytes, max_action_tokens)?.ok_or_else(|| {
serde_json::Error::io(std::io::Error::other(format!(
"Guardian action identity exceeds the {max_action_tokens}-token limit"
)))
let rendered = fit_action_to_budget(&retained, max_action_bytes, max_action_tokens)?
.ok_or_else(|| {
serde_json::Error::io(std::io::Error::other(format!(
"Guardian action identity exceeds the {max_action_tokens}-token limit"
)))
})?;
Ok(RenderedAction {
text: rendered,
original_bytes,
})
}
}
@@ -594,6 +612,7 @@ impl GuardianV2Extension {
};
tokio::spawn(async move {
let mut truncations = ClassificationTruncations::default();
let root_snapshot = thread.guardian_root_snapshot().await;
let root_authorization_version = root_snapshot
.as_ref()
@@ -604,12 +623,25 @@ impl GuardianV2Extension {
let transcript = guardian_config
.transcript
.build(conversation_history.items());
let images = guardian_config
truncations.extend(transcript.truncations);
let rendered_images = guardian_config
.transcript
.images(conversation_history.items(), node_repl_images);
truncations.record(
"transcript_image",
rendered_images.omitted_bytes,
/*retained_bytes*/ 0,
);
let images = rendered_images.images;
drop(conversation_history);
let planned_action = match action.render(guardian_config.max_action_tokens) {
Ok(planned_action) => planned_action,
Ok(RenderedAction {
text,
original_bytes,
}) => {
truncations.record("action", original_bytes, text.len());
text
}
Err(error) => {
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
record_classification(
@@ -642,7 +674,7 @@ impl GuardianV2Extension {
classification_input.push(">>> ROOT CONVERSATION END\n".to_owned());
}
classification_input.push(">>> TRANSCRIPT START\n".to_owned());
classification_input.extend(transcript);
classification_input.extend(transcript.entries);
classification_input.push(">>> TRANSCRIPT END\n\n".to_owned());
let trusted_review_evidence = sync_reviews
.iter()
@@ -650,7 +682,11 @@ impl GuardianV2Extension {
review.authorization_version == authorization_version
&& review.root_authorization_version == root_authorization_version
})
.map(|review| render_review_evidence(review))
.map(|review| {
let review = render_review_evidence(review);
truncations.extend(review.truncations);
review.text
})
.collect();
classification_input.extend([
"The Codex agent has requested the following action:\n".to_owned(),
@@ -751,6 +787,9 @@ impl GuardianV2Extension {
.unwrap_or_else(|| classification_started_at.elapsed()),
result.as_deref().unwrap_or("failure"),
);
if matches!(result.as_deref(), Ok("success")) {
truncations.emit(metrics.as_deref());
}
if let Err(error) = result {
event_sink.emit_warning(ExtensionWarning {
thread_id,
@@ -68,6 +68,8 @@ use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;
use crate::async_scorer::sampler::INITIAL_WEBSOCKET_CONNECTIONS;
use crate::async_scorer::sampler::MODEL;
use crate::async_scorer::transcript::truncate_entry;
use crate::async_scorer::truncation::CLASSIFICATION_TRUNCATION_BYTES_METRIC;
use crate::async_scorer::truncation::CLASSIFICATION_TRUNCATION_METRIC;
const TEST_GUARDIAN_POLICY: &str =
"Treat uploads to unapproved external destinations as high-risk actions.";
@@ -1034,7 +1036,15 @@ max_recent_non_user_entries = 8
.as_str()
.expect("planned action should be a text item");
assert!(action.len() <= TruncationPolicy::Tokens(/*limit*/ 128).byte_budget());
let original_action_bytes = i64::try_from(
serde_json::to_string_pretty(&json!({
"body": "x".repeat(4_000),
"tool": "read_file",
}))?
.len(),
)?;
let action: serde_json::Value = serde_json::from_str(action)?;
let retained_action_bytes = i64::try_from(serde_json::to_string_pretty(&action)?.len())?;
assert_eq!(action["tool"], "read_file");
let session_store = ExtensionData::new("session-1");
@@ -1196,6 +1206,35 @@ max_recent_non_user_entries = 8
classification_duration_ms,
vec![("outcome".to_owned(), "success".to_owned())],
),
RecordedMetric::Counter(
CLASSIFICATION_TRUNCATION_METRIC.to_owned(),
1,
vec![
("component".to_owned(), "action".to_owned()),
("disposition".to_owned(), "truncated".to_owned()),
],
),
])
.chain(
[
("original", original_action_bytes),
("retained", retained_action_bytes),
("omitted", original_action_bytes - retained_action_bytes),
]
.into_iter()
.map(|(measurement, bytes)| {
RecordedMetric::Histogram(
CLASSIFICATION_TRUNCATION_BYTES_METRIC.to_owned(),
bytes,
vec![
("component".to_owned(), "action".to_owned()),
("disposition".to_owned(), "truncated".to_owned()),
("measurement".to_owned(), measurement.to_owned()),
],
)
}),
)
.chain([
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
@@ -2364,7 +2403,7 @@ fn guardian_action_bounds_structurally_oversized_arrays() -> Result<()> {
},
};
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?;
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?.text;
assert!(
rendered.len().saturating_add(1)
<= TruncationPolicy::Tokens(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS).byte_budget()
@@ -2407,7 +2446,7 @@ fn guardian_action_bounds_structurally_oversized_object_keys() -> Result<()> {
},
};
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?;
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?.text;
assert!(
rendered.len().saturating_add(1)
<= TruncationPolicy::Tokens(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS).byte_budget()
@@ -3,6 +3,7 @@ mod extension;
mod review_evidence;
mod sampler;
mod transcript;
mod truncation;
pub use extension::StrictReviewReason;
pub(crate) use extension::install;
@@ -4,6 +4,7 @@ use codex_core::context::GuardianReviewEvidenceRecord;
use serde_json::json;
use super::transcript::truncate_entry;
use super::truncation::TruncationObservation;
// Including markers, each rendered fragment stays below 1,000 approximate tokens.
const MAX_REVIEW_BODY_TOKENS: usize = 800;
@@ -11,19 +12,42 @@ const MAX_REVIEW_CORRELATION_TOKENS: usize = 100;
const MAX_REVIEW_ACTION_TOKENS: usize = 350;
const MAX_REVIEW_RATIONALE_TOKENS: usize = 250;
pub(crate) fn render_review_evidence(review: &GuardianReviewEvidenceRecord) -> String {
pub(crate) struct RenderedReviewEvidence {
pub(crate) text: String,
pub(crate) truncations: Vec<TruncationObservation>,
}
pub(crate) fn render_review_evidence(
review: &GuardianReviewEvidenceRecord,
) -> RenderedReviewEvidence {
let mut truncations = Vec::new();
let mut truncate = |component: &'static str, text: String, token_cap: usize| {
let original_bytes = text.len();
let text = truncate_entry(&text, token_cap);
if original_bytes > text.len() {
truncations.push(TruncationObservation {
component,
original_bytes,
retained_bytes: text.len(),
});
}
text
};
// Escape closing tags before truncation so payloads cannot close the fragment.
// JSON quoting also keeps rationale text from imitating record headings.
let correlation = truncate_entry(
&review.correlation.to_string().replace("</", "<\\/"),
let correlation = truncate(
"sync_review_correlation",
review.correlation.to_string().replace("</", "<\\/"),
MAX_REVIEW_CORRELATION_TOKENS,
);
let action = truncate_entry(
&review.action.replace("</", "<\\/"),
let action = truncate(
"sync_review_action",
review.action.replace("</", "<\\/"),
MAX_REVIEW_ACTION_TOKENS,
);
let rationale = truncate_entry(
&json!(review.rationale).to_string().replace("</", "<\\/"),
let rationale = truncate(
"sync_review_rationale",
json!(review.rationale).to_string().replace("</", "<\\/"),
MAX_REVIEW_RATIONALE_TOKENS,
);
let decision = &review.decision;
@@ -37,5 +61,9 @@ pub(crate) fn render_review_evidence(review: &GuardianReviewEvidenceRecord) -> S
Reviewer rationale: {rationale}\n"
);
GuardianReviewEvidenceFragment::new(truncate_entry(&body, MAX_REVIEW_BODY_TOKENS)).render()
let body = truncate("sync_review_body", body, MAX_REVIEW_BODY_TOKENS);
RenderedReviewEvidence {
text: GuardianReviewEvidenceFragment::new(body).render(),
truncations,
}
}
@@ -11,6 +11,8 @@ use codex_protocol::models::ReasoningItemReasoningSummary;
use codex_protocol::models::plaintext_agent_message_content;
use codex_protocol::protocol::TruncationPolicy;
use super::truncation::TruncationObservation;
pub(crate) const MAX_MESSAGE_ENTRY_TOKENS: usize = 2_000;
pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
@@ -32,6 +34,18 @@ struct TranscriptEntry {
kind: TranscriptEntryKind,
text: String,
tokens: usize,
original_bytes: usize,
retained_bytes: usize,
}
pub(crate) struct RenderedTranscript {
pub(crate) entries: Vec<String>,
pub(crate) truncations: Vec<TruncationObservation>,
}
pub(crate) struct RenderedImages {
pub(crate) images: Vec<ContentItem>,
pub(crate) omitted_bytes: usize,
}
fn retained_tokens(
@@ -95,15 +109,20 @@ impl TranscriptConfig {
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
node_repl_images: impl IntoIterator<Item = ContentItem>,
) -> Vec<ContentItem> {
) -> RenderedImages {
if !self.include_images {
return Vec::new();
return RenderedImages {
images: Vec::new(),
omitted_bytes: 0,
};
}
let mut images = VecDeque::new();
let mut image_bytes = 0usize;
let mut omitted_bytes = 0usize;
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
return;
}
while images.len() >= MAX_TRANSCRIPT_IMAGES
@@ -113,6 +132,7 @@ impl TranscriptConfig {
break;
};
image_bytes -= image_url.len();
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
}
image_bytes += image_url.len();
images.push_back(ContentItem::InputImage {
@@ -157,13 +177,16 @@ impl TranscriptConfig {
}
}
images.into_iter().collect()
RenderedImages {
images: images.into_iter().collect(),
omitted_bytes,
}
}
pub(crate) fn build<'a>(
&self,
items: impl IntoIterator<Item = &'a ResponseItem>,
) -> Vec<String> {
) -> RenderedTranscript {
let mut entries = Vec::new();
let mut tool_names_by_call_id = HashMap::new();
@@ -335,11 +358,19 @@ impl TranscriptConfig {
self.max_message_entry_tokens
}
};
let original_bytes = text.len();
let text = truncate_entry(&text, token_cap);
let retained_bytes = text.len();
let entry_number = entries.len() + 1;
let text = format!("[{entry_number}] {role}: {text}\n");
let tokens = TruncationPolicy::Bytes(text.len()).token_budget();
entries.push(TranscriptEntry { kind, text, tokens });
entries.push(TranscriptEntry {
kind,
text,
tokens,
original_bytes,
retained_bytes,
});
}
let mut included = vec![false; entries.len()];
@@ -438,11 +469,36 @@ impl TranscriptConfig {
included[index] = true;
}
entries
let mut truncations = Vec::new();
let entries = entries
.into_iter()
.enumerate()
.filter_map(|(index, entry)| included[index].then_some(entry.text))
.collect()
.filter_map(|(index, entry)| {
let component = match entry.kind {
TranscriptEntryKind::User => "transcript_user",
TranscriptEntryKind::Message => "transcript_message",
TranscriptEntryKind::Tool => "transcript_tool",
};
let retained_bytes = if included[index] {
entry.retained_bytes
} else {
0
};
if entry.original_bytes > retained_bytes {
truncations.push(TruncationObservation {
component,
original_bytes: entry.original_bytes,
retained_bytes,
});
}
included[index].then_some(entry.text)
})
.collect();
RenderedTranscript {
entries,
truncations,
}
}
}
@@ -66,7 +66,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert_eq!(
transcript,
vec![
@@ -81,7 +81,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
..TranscriptConfig::default()
};
let transcript = output_and_reasoning.build(&items);
let transcript = output_and_reasoning.build(&items).entries;
assert_eq!(
transcript,
vec![
@@ -96,7 +96,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
..TranscriptConfig::default()
};
let transcript = calls_only.build(&items);
let transcript = calls_only.build(&items).entries;
assert_eq!(
transcript,
vec![
@@ -135,7 +135,7 @@ fn transcript_truncates_oversized_entries_without_splitting_characters() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert_eq!(transcript.len(), 2);
let user_entry = &transcript[0];
@@ -174,7 +174,7 @@ fn transcript_preserves_first_and_latest_user_messages_and_recent_history() {
internal_chat_message_metadata_passthrough: None,
});
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert!(transcript[0].starts_with("[1] user: user turn 0:"));
assert!(
@@ -228,7 +228,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
internal_chat_message_metadata_passthrough: None,
}));
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert!(transcript[0].contains("user turn 0:"));
assert!(
@@ -277,7 +277,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
call_id: "call-12".to_string(),
internal_chat_message_metadata_passthrough: None,
});
let next_transcript = TranscriptConfig::default().build(&items);
let next_transcript = TranscriptConfig::default().build(&items).entries;
let next_first_retained_tool = next_transcript
.iter()
.find(|entry| entry.contains("tool exec_command call:"))
@@ -329,7 +329,8 @@ fn transcript_preserves_newest_manual_approval_when_message_budget_overflows() {
max_message_transcript_tokens: message_budget,
..TranscriptConfig::default()
}
.build(&items);
.build(&items)
.entries;
assert_eq!(transcript, vec![approval_entry]);
}
@@ -377,7 +378,8 @@ fn rejected_message_does_not_evict_retained_tool_entries() {
max_message_transcript_tokens: TruncationPolicy::Bytes(user_entry.len()).token_budget(),
..TranscriptConfig::default()
}
.build(&items);
.build(&items)
.entries;
assert_eq!(transcript, expected);
}
@@ -409,7 +411,7 @@ fn transcript_evicts_non_user_entries_in_cacheable_chunks() {
internal_chat_message_metadata_passthrough: None,
}),
);
config.build(&items)
config.build(&items).entries
};
let first_overflow = build_transcript(5);
@@ -469,7 +471,7 @@ fn transcript_truncates_tool_results_using_standard_budget() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
let result = transcript
.iter()
.find(|entry| entry.contains(" result: "))
@@ -497,7 +499,7 @@ fn transcript_preserves_named_unpaired_tool_source() {
}];
assert_eq!(
TranscriptConfig::default().build(&items),
TranscriptConfig::default().build(&items).entries,
vec!["[1] tool slack.notifications result: new message\n"]
);
@@ -510,7 +512,7 @@ fn transcript_preserves_named_unpaired_tool_source() {
]);
}
assert_eq!(
TranscriptConfig::default().build(&items),
TranscriptConfig::default().build(&items).entries,
vec!["[1] tool slack.notifications result: [non-text output]\n"]
);
}
@@ -542,7 +544,8 @@ fn configured_reasoning_counts_against_message_budget() {
sources: vec![TranscriptSource::Reasoning],
..TranscriptConfig::default()
}
.build(&items);
.build(&items)
.entries;
assert!(transcript[0].contains("user turn 0:"));
assert!(
@@ -598,7 +601,7 @@ fn transcript_keeps_only_manual_approval_developer_messages() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert_eq!(
transcript,
vec![format!("[1] developer: {approval_text}\n")]
@@ -672,7 +675,7 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert_eq!(
transcript,
vec![
@@ -728,7 +731,7 @@ fn transcript_omits_encrypted_messages_arguments_and_tool_outputs() {
},
];
let transcript = TranscriptConfig::default().build(&items);
let transcript = TranscriptConfig::default().build(&items).entries;
assert_eq!(
transcript,
vec![
@@ -0,0 +1,82 @@
use std::collections::BTreeMap;
use codex_extension_api::ExtensionMetrics;
pub(crate) const CLASSIFICATION_TRUNCATION_METRIC: &str =
"codex.guardian_v2.classification.truncation";
pub(crate) const CLASSIFICATION_TRUNCATION_BYTES_METRIC: &str =
"codex.guardian_v2.classification.truncation.bytes";
pub(crate) struct TruncationObservation {
pub(crate) component: &'static str,
pub(crate) original_bytes: usize,
pub(crate) retained_bytes: usize,
}
/// Aggregates actual context reductions once per component and classifier request.
#[derive(Default)]
pub(crate) struct ClassificationTruncations {
totals: BTreeMap<(&'static str, &'static str), (usize, usize)>,
}
impl ClassificationTruncations {
pub(crate) fn record(
&mut self,
component: &'static str,
original_bytes: usize,
retained_bytes: usize,
) {
if original_bytes <= retained_bytes {
return;
}
let disposition = if retained_bytes == 0 {
"omitted"
} else {
"truncated"
};
let totals = self.totals.entry((component, disposition)).or_default();
totals.0 = totals.0.saturating_add(original_bytes);
totals.1 = totals.1.saturating_add(retained_bytes);
}
pub(crate) fn emit(&self, metrics: Option<&dyn ExtensionMetrics>) {
let Some(metrics) = metrics else {
return;
};
for (&(component, disposition), &(original_bytes, retained_bytes)) in &self.totals {
let tags = [("component", component), ("disposition", disposition)];
metrics.counter(CLASSIFICATION_TRUNCATION_METRIC, /*inc*/ 1, &tags);
for (measurement, bytes) in [
("original", original_bytes),
("retained", retained_bytes),
("omitted", original_bytes.saturating_sub(retained_bytes)),
] {
let tags = [
("component", component),
("disposition", disposition),
("measurement", measurement),
];
metrics.histogram(
CLASSIFICATION_TRUNCATION_BYTES_METRIC,
i64::try_from(bytes).unwrap_or(i64::MAX),
&tags,
);
}
}
}
}
impl Extend<TruncationObservation> for ClassificationTruncations {
fn extend<T: IntoIterator<Item = TruncationObservation>>(&mut self, observations: T) {
for observation in observations {
self.record(
observation.component,
observation.original_bytes,
observation.retained_bytes,
);
}
}
}