mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
Record Guardian classifier input truncation metrics (#40465)
## What changed - Track truncation and omission across Guardian v2 actions, transcript entries and images, and synchronous review evidence. - Emit `codex.guardian_v2.classification.truncation` counters and byte histograms for original, retained, and omitted content after successful classifications. - Tag observations by input component and whether content was truncated or fully omitted. ## Testing - Extend async scorer coverage for action truncation metrics. - Verify the app-server Guardian flow records truncated review evidence. GitOrigin-RevId: 70680897f75716770aa072f866d60bd37e171c1d
This commit is contained in:
@@ -70,6 +70,7 @@ struct MockResponsesState {
|
||||
allow_luna: Notify,
|
||||
allow_guardian_review: Notify,
|
||||
classification_completed: Notify,
|
||||
truncation_recorded: Notify,
|
||||
luna_score: f64,
|
||||
review_outcome: ReviewOutcome,
|
||||
transcript_content: TranscriptContent,
|
||||
@@ -356,6 +357,11 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
if body.contains("codex.guardian_v2.classification") {
|
||||
state.classification_completed.notify_one();
|
||||
}
|
||||
if body.contains("codex.guardian_v2.classification.truncation")
|
||||
&& body.contains("sync_review_action")
|
||||
{
|
||||
state.truncation_recorded.notify_one();
|
||||
}
|
||||
},
|
||||
),
|
||||
)
|
||||
@@ -747,6 +753,10 @@ async fn guardian_v2_routes_tool_approvals(
|
||||
responses_state.allow_luna.notify_one();
|
||||
}
|
||||
|
||||
if matches!(review_outcome, ReviewOutcome::Deny) {
|
||||
timeout(TIMEOUT, responses_state.truncation_recorded.notified()).await?;
|
||||
}
|
||||
|
||||
mcp_server_handle.abort();
|
||||
responses_server.abort();
|
||||
Ok(())
|
||||
|
||||
@@ -47,12 +47,18 @@ use super::sampler::LunaSamplerConfig;
|
||||
use super::sampler::LunaSamplerError;
|
||||
use super::sampler::LunaSamplingRequest;
|
||||
use super::sampler::MODEL;
|
||||
use super::truncation::ClassificationTruncations;
|
||||
|
||||
struct GuardianAction {
|
||||
tool_name: ToolName,
|
||||
payload: ToolPayload,
|
||||
}
|
||||
|
||||
struct RenderedAction {
|
||||
text: String,
|
||||
original_bytes: usize,
|
||||
}
|
||||
|
||||
fn should_classify_tool(
|
||||
tool_name: &ToolName,
|
||||
payload: &ToolPayload,
|
||||
@@ -80,7 +86,7 @@ fn should_classify_tool(
|
||||
}
|
||||
|
||||
impl GuardianAction {
|
||||
fn render(self, max_action_tokens: usize) -> serde_json::Result<String> {
|
||||
fn render(self, max_action_tokens: usize) -> serde_json::Result<RenderedAction> {
|
||||
let arguments = match self.payload {
|
||||
ToolPayload::Function { arguments } => {
|
||||
serde_json::from_str(&arguments).unwrap_or(serde_json::Value::String(arguments))
|
||||
@@ -103,13 +109,20 @@ impl GuardianAction {
|
||||
.for_each(serde_json::Value::sort_all_objects);
|
||||
let max_action_bytes = TruncationPolicy::Tokens(max_action_tokens).byte_budget();
|
||||
let rendered = serde_json::to_string_pretty(&action)?;
|
||||
let original_bytes = rendered.len();
|
||||
if rendered.len().saturating_add(1) <= max_action_bytes {
|
||||
return Ok(rendered);
|
||||
return Ok(RenderedAction {
|
||||
text: rendered,
|
||||
original_bytes,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(rendered) = fit_action_to_budget(&action, max_action_bytes, max_action_tokens)?
|
||||
{
|
||||
return Ok(rendered);
|
||||
return Ok(RenderedAction {
|
||||
text: rendered,
|
||||
original_bytes,
|
||||
});
|
||||
}
|
||||
|
||||
let mut omission_key = "_guardian_omitted_fields".to_owned();
|
||||
@@ -148,10 +161,15 @@ impl GuardianAction {
|
||||
}
|
||||
|
||||
retained.sort_keys();
|
||||
fit_action_to_budget(&retained, max_action_bytes, max_action_tokens)?.ok_or_else(|| {
|
||||
serde_json::Error::io(std::io::Error::other(format!(
|
||||
"Guardian action identity exceeds the {max_action_tokens}-token limit"
|
||||
)))
|
||||
let rendered = fit_action_to_budget(&retained, max_action_bytes, max_action_tokens)?
|
||||
.ok_or_else(|| {
|
||||
serde_json::Error::io(std::io::Error::other(format!(
|
||||
"Guardian action identity exceeds the {max_action_tokens}-token limit"
|
||||
)))
|
||||
})?;
|
||||
Ok(RenderedAction {
|
||||
text: rendered,
|
||||
original_bytes,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -594,6 +612,7 @@ impl GuardianV2Extension {
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut truncations = ClassificationTruncations::default();
|
||||
let root_snapshot = thread.guardian_root_snapshot().await;
|
||||
let root_authorization_version = root_snapshot
|
||||
.as_ref()
|
||||
@@ -604,12 +623,25 @@ impl GuardianV2Extension {
|
||||
let transcript = guardian_config
|
||||
.transcript
|
||||
.build(conversation_history.items());
|
||||
let images = guardian_config
|
||||
truncations.extend(transcript.truncations);
|
||||
let rendered_images = guardian_config
|
||||
.transcript
|
||||
.images(conversation_history.items(), node_repl_images);
|
||||
truncations.record(
|
||||
"transcript_image",
|
||||
rendered_images.omitted_bytes,
|
||||
/*retained_bytes*/ 0,
|
||||
);
|
||||
let images = rendered_images.images;
|
||||
drop(conversation_history);
|
||||
let planned_action = match action.render(guardian_config.max_action_tokens) {
|
||||
Ok(planned_action) => planned_action,
|
||||
Ok(RenderedAction {
|
||||
text,
|
||||
original_bytes,
|
||||
}) => {
|
||||
truncations.record("action", original_bytes, text.len());
|
||||
text
|
||||
}
|
||||
Err(error) => {
|
||||
Self::record_fail_closed_score(thread.thread_extension_data(), sampled_at);
|
||||
record_classification(
|
||||
@@ -642,7 +674,7 @@ impl GuardianV2Extension {
|
||||
classification_input.push(">>> ROOT CONVERSATION END\n".to_owned());
|
||||
}
|
||||
classification_input.push(">>> TRANSCRIPT START\n".to_owned());
|
||||
classification_input.extend(transcript);
|
||||
classification_input.extend(transcript.entries);
|
||||
classification_input.push(">>> TRANSCRIPT END\n\n".to_owned());
|
||||
let trusted_review_evidence = sync_reviews
|
||||
.iter()
|
||||
@@ -650,7 +682,11 @@ impl GuardianV2Extension {
|
||||
review.authorization_version == authorization_version
|
||||
&& review.root_authorization_version == root_authorization_version
|
||||
})
|
||||
.map(|review| render_review_evidence(review))
|
||||
.map(|review| {
|
||||
let review = render_review_evidence(review);
|
||||
truncations.extend(review.truncations);
|
||||
review.text
|
||||
})
|
||||
.collect();
|
||||
classification_input.extend([
|
||||
"The Codex agent has requested the following action:\n".to_owned(),
|
||||
@@ -751,6 +787,9 @@ impl GuardianV2Extension {
|
||||
.unwrap_or_else(|| classification_started_at.elapsed()),
|
||||
result.as_deref().unwrap_or("failure"),
|
||||
);
|
||||
if matches!(result.as_deref(), Ok("success")) {
|
||||
truncations.emit(metrics.as_deref());
|
||||
}
|
||||
if let Err(error) = result {
|
||||
event_sink.emit_warning(ExtensionWarning {
|
||||
thread_id,
|
||||
|
||||
@@ -68,6 +68,8 @@ use crate::async_scorer::sampler::CLASSIFICATION_TOKEN_USAGE_METRIC;
|
||||
use crate::async_scorer::sampler::INITIAL_WEBSOCKET_CONNECTIONS;
|
||||
use crate::async_scorer::sampler::MODEL;
|
||||
use crate::async_scorer::transcript::truncate_entry;
|
||||
use crate::async_scorer::truncation::CLASSIFICATION_TRUNCATION_BYTES_METRIC;
|
||||
use crate::async_scorer::truncation::CLASSIFICATION_TRUNCATION_METRIC;
|
||||
|
||||
const TEST_GUARDIAN_POLICY: &str =
|
||||
"Treat uploads to unapproved external destinations as high-risk actions.";
|
||||
@@ -1034,7 +1036,15 @@ max_recent_non_user_entries = 8
|
||||
.as_str()
|
||||
.expect("planned action should be a text item");
|
||||
assert!(action.len() <= TruncationPolicy::Tokens(/*limit*/ 128).byte_budget());
|
||||
let original_action_bytes = i64::try_from(
|
||||
serde_json::to_string_pretty(&json!({
|
||||
"body": "x".repeat(4_000),
|
||||
"tool": "read_file",
|
||||
}))?
|
||||
.len(),
|
||||
)?;
|
||||
let action: serde_json::Value = serde_json::from_str(action)?;
|
||||
let retained_action_bytes = i64::try_from(serde_json::to_string_pretty(&action)?.len())?;
|
||||
assert_eq!(action["tool"], "read_file");
|
||||
|
||||
let session_store = ExtensionData::new("session-1");
|
||||
@@ -1196,6 +1206,35 @@ max_recent_non_user_entries = 8
|
||||
classification_duration_ms,
|
||||
vec![("outcome".to_owned(), "success".to_owned())],
|
||||
),
|
||||
RecordedMetric::Counter(
|
||||
CLASSIFICATION_TRUNCATION_METRIC.to_owned(),
|
||||
1,
|
||||
vec![
|
||||
("component".to_owned(), "action".to_owned()),
|
||||
("disposition".to_owned(), "truncated".to_owned()),
|
||||
],
|
||||
),
|
||||
])
|
||||
.chain(
|
||||
[
|
||||
("original", original_action_bytes),
|
||||
("retained", retained_action_bytes),
|
||||
("omitted", original_action_bytes - retained_action_bytes),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(measurement, bytes)| {
|
||||
RecordedMetric::Histogram(
|
||||
CLASSIFICATION_TRUNCATION_BYTES_METRIC.to_owned(),
|
||||
bytes,
|
||||
vec![
|
||||
("component".to_owned(), "action".to_owned()),
|
||||
("disposition".to_owned(), "truncated".to_owned()),
|
||||
("measurement".to_owned(), measurement.to_owned()),
|
||||
],
|
||||
)
|
||||
}),
|
||||
)
|
||||
.chain([
|
||||
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
|
||||
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 0, vec![]),
|
||||
RecordedMetric::Histogram(TOOL_CALL_LAG_METRIC.to_owned(), 2, vec![]),
|
||||
@@ -2364,7 +2403,7 @@ fn guardian_action_bounds_structurally_oversized_arrays() -> Result<()> {
|
||||
},
|
||||
};
|
||||
|
||||
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?;
|
||||
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?.text;
|
||||
assert!(
|
||||
rendered.len().saturating_add(1)
|
||||
<= TruncationPolicy::Tokens(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS).byte_budget()
|
||||
@@ -2407,7 +2446,7 @@ fn guardian_action_bounds_structurally_oversized_object_keys() -> Result<()> {
|
||||
},
|
||||
};
|
||||
|
||||
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?;
|
||||
let rendered = action.render(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS)?.text;
|
||||
assert!(
|
||||
rendered.len().saturating_add(1)
|
||||
<= TruncationPolicy::Tokens(DEFAULT_MODEL_CONTEXT_ITEM_TOKENS).byte_budget()
|
||||
|
||||
@@ -3,6 +3,7 @@ mod extension;
|
||||
mod review_evidence;
|
||||
mod sampler;
|
||||
mod transcript;
|
||||
mod truncation;
|
||||
|
||||
pub use extension::StrictReviewReason;
|
||||
pub(crate) use extension::install;
|
||||
|
||||
@@ -4,6 +4,7 @@ use codex_core::context::GuardianReviewEvidenceRecord;
|
||||
use serde_json::json;
|
||||
|
||||
use super::transcript::truncate_entry;
|
||||
use super::truncation::TruncationObservation;
|
||||
|
||||
// Including markers, each rendered fragment stays below 1,000 approximate tokens.
|
||||
const MAX_REVIEW_BODY_TOKENS: usize = 800;
|
||||
@@ -11,19 +12,42 @@ const MAX_REVIEW_CORRELATION_TOKENS: usize = 100;
|
||||
const MAX_REVIEW_ACTION_TOKENS: usize = 350;
|
||||
const MAX_REVIEW_RATIONALE_TOKENS: usize = 250;
|
||||
|
||||
pub(crate) fn render_review_evidence(review: &GuardianReviewEvidenceRecord) -> String {
|
||||
pub(crate) struct RenderedReviewEvidence {
|
||||
pub(crate) text: String,
|
||||
pub(crate) truncations: Vec<TruncationObservation>,
|
||||
}
|
||||
|
||||
pub(crate) fn render_review_evidence(
|
||||
review: &GuardianReviewEvidenceRecord,
|
||||
) -> RenderedReviewEvidence {
|
||||
let mut truncations = Vec::new();
|
||||
let mut truncate = |component: &'static str, text: String, token_cap: usize| {
|
||||
let original_bytes = text.len();
|
||||
let text = truncate_entry(&text, token_cap);
|
||||
if original_bytes > text.len() {
|
||||
truncations.push(TruncationObservation {
|
||||
component,
|
||||
original_bytes,
|
||||
retained_bytes: text.len(),
|
||||
});
|
||||
}
|
||||
text
|
||||
};
|
||||
// Escape closing tags before truncation so payloads cannot close the fragment.
|
||||
// JSON quoting also keeps rationale text from imitating record headings.
|
||||
let correlation = truncate_entry(
|
||||
&review.correlation.to_string().replace("</", "<\\/"),
|
||||
let correlation = truncate(
|
||||
"sync_review_correlation",
|
||||
review.correlation.to_string().replace("</", "<\\/"),
|
||||
MAX_REVIEW_CORRELATION_TOKENS,
|
||||
);
|
||||
let action = truncate_entry(
|
||||
&review.action.replace("</", "<\\/"),
|
||||
let action = truncate(
|
||||
"sync_review_action",
|
||||
review.action.replace("</", "<\\/"),
|
||||
MAX_REVIEW_ACTION_TOKENS,
|
||||
);
|
||||
let rationale = truncate_entry(
|
||||
&json!(review.rationale).to_string().replace("</", "<\\/"),
|
||||
let rationale = truncate(
|
||||
"sync_review_rationale",
|
||||
json!(review.rationale).to_string().replace("</", "<\\/"),
|
||||
MAX_REVIEW_RATIONALE_TOKENS,
|
||||
);
|
||||
let decision = &review.decision;
|
||||
@@ -37,5 +61,9 @@ pub(crate) fn render_review_evidence(review: &GuardianReviewEvidenceRecord) -> S
|
||||
Reviewer rationale: {rationale}\n"
|
||||
);
|
||||
|
||||
GuardianReviewEvidenceFragment::new(truncate_entry(&body, MAX_REVIEW_BODY_TOKENS)).render()
|
||||
let body = truncate("sync_review_body", body, MAX_REVIEW_BODY_TOKENS);
|
||||
RenderedReviewEvidence {
|
||||
text: GuardianReviewEvidenceFragment::new(body).render(),
|
||||
truncations,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ use codex_protocol::models::ReasoningItemReasoningSummary;
|
||||
use codex_protocol::models::plaintext_agent_message_content;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
|
||||
use super::truncation::TruncationObservation;
|
||||
|
||||
pub(crate) const MAX_MESSAGE_ENTRY_TOKENS: usize = 2_000;
|
||||
pub(crate) const MAX_TOOL_ENTRY_TOKENS: usize = 1_000;
|
||||
pub(crate) const MAX_MESSAGE_TRANSCRIPT_TOKENS: usize = 10_000;
|
||||
@@ -32,6 +34,18 @@ struct TranscriptEntry {
|
||||
kind: TranscriptEntryKind,
|
||||
text: String,
|
||||
tokens: usize,
|
||||
original_bytes: usize,
|
||||
retained_bytes: usize,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedTranscript {
|
||||
pub(crate) entries: Vec<String>,
|
||||
pub(crate) truncations: Vec<TruncationObservation>,
|
||||
}
|
||||
|
||||
pub(crate) struct RenderedImages {
|
||||
pub(crate) images: Vec<ContentItem>,
|
||||
pub(crate) omitted_bytes: usize,
|
||||
}
|
||||
|
||||
fn retained_tokens(
|
||||
@@ -95,15 +109,20 @@ impl TranscriptConfig {
|
||||
&self,
|
||||
items: impl IntoIterator<Item = &'a ResponseItem>,
|
||||
node_repl_images: impl IntoIterator<Item = ContentItem>,
|
||||
) -> Vec<ContentItem> {
|
||||
) -> RenderedImages {
|
||||
if !self.include_images {
|
||||
return Vec::new();
|
||||
return RenderedImages {
|
||||
images: Vec::new(),
|
||||
omitted_bytes: 0,
|
||||
};
|
||||
}
|
||||
|
||||
let mut images = VecDeque::new();
|
||||
let mut image_bytes = 0usize;
|
||||
let mut omitted_bytes = 0usize;
|
||||
let mut include_image = |image_url: &str, detail: Option<ImageDetail>| {
|
||||
if image_url.len() > MAX_TRANSCRIPT_IMAGE_BYTES {
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
return;
|
||||
}
|
||||
while images.len() >= MAX_TRANSCRIPT_IMAGES
|
||||
@@ -113,6 +132,7 @@ impl TranscriptConfig {
|
||||
break;
|
||||
};
|
||||
image_bytes -= image_url.len();
|
||||
omitted_bytes = omitted_bytes.saturating_add(image_url.len());
|
||||
}
|
||||
image_bytes += image_url.len();
|
||||
images.push_back(ContentItem::InputImage {
|
||||
@@ -157,13 +177,16 @@ impl TranscriptConfig {
|
||||
}
|
||||
}
|
||||
|
||||
images.into_iter().collect()
|
||||
RenderedImages {
|
||||
images: images.into_iter().collect(),
|
||||
omitted_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn build<'a>(
|
||||
&self,
|
||||
items: impl IntoIterator<Item = &'a ResponseItem>,
|
||||
) -> Vec<String> {
|
||||
) -> RenderedTranscript {
|
||||
let mut entries = Vec::new();
|
||||
let mut tool_names_by_call_id = HashMap::new();
|
||||
|
||||
@@ -335,11 +358,19 @@ impl TranscriptConfig {
|
||||
self.max_message_entry_tokens
|
||||
}
|
||||
};
|
||||
let original_bytes = text.len();
|
||||
let text = truncate_entry(&text, token_cap);
|
||||
let retained_bytes = text.len();
|
||||
let entry_number = entries.len() + 1;
|
||||
let text = format!("[{entry_number}] {role}: {text}\n");
|
||||
let tokens = TruncationPolicy::Bytes(text.len()).token_budget();
|
||||
entries.push(TranscriptEntry { kind, text, tokens });
|
||||
entries.push(TranscriptEntry {
|
||||
kind,
|
||||
text,
|
||||
tokens,
|
||||
original_bytes,
|
||||
retained_bytes,
|
||||
});
|
||||
}
|
||||
|
||||
let mut included = vec![false; entries.len()];
|
||||
@@ -438,11 +469,36 @@ impl TranscriptConfig {
|
||||
included[index] = true;
|
||||
}
|
||||
|
||||
entries
|
||||
let mut truncations = Vec::new();
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, entry)| included[index].then_some(entry.text))
|
||||
.collect()
|
||||
.filter_map(|(index, entry)| {
|
||||
let component = match entry.kind {
|
||||
TranscriptEntryKind::User => "transcript_user",
|
||||
TranscriptEntryKind::Message => "transcript_message",
|
||||
TranscriptEntryKind::Tool => "transcript_tool",
|
||||
};
|
||||
let retained_bytes = if included[index] {
|
||||
entry.retained_bytes
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if entry.original_bytes > retained_bytes {
|
||||
truncations.push(TruncationObservation {
|
||||
component,
|
||||
original_bytes: entry.original_bytes,
|
||||
retained_bytes,
|
||||
});
|
||||
}
|
||||
included[index].then_some(entry.text)
|
||||
})
|
||||
.collect();
|
||||
|
||||
RenderedTranscript {
|
||||
entries,
|
||||
truncations,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![
|
||||
@@ -81,7 +81,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
..TranscriptConfig::default()
|
||||
};
|
||||
|
||||
let transcript = output_and_reasoning.build(&items);
|
||||
let transcript = output_and_reasoning.build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![
|
||||
@@ -96,7 +96,7 @@ fn transcript_keeps_conversation_and_configured_sources() {
|
||||
..TranscriptConfig::default()
|
||||
};
|
||||
|
||||
let transcript = calls_only.build(&items);
|
||||
let transcript = calls_only.build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![
|
||||
@@ -135,7 +135,7 @@ fn transcript_truncates_oversized_entries_without_splitting_characters() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
|
||||
assert_eq!(transcript.len(), 2);
|
||||
let user_entry = &transcript[0];
|
||||
@@ -174,7 +174,7 @@ fn transcript_preserves_first_and_latest_user_messages_and_recent_history() {
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
});
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
|
||||
assert!(transcript[0].starts_with("[1] user: user turn 0:"));
|
||||
assert!(
|
||||
@@ -228,7 +228,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
}));
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
|
||||
assert!(transcript[0].contains("user turn 0:"));
|
||||
assert!(
|
||||
@@ -277,7 +277,7 @@ fn transcript_reserves_separate_budget_for_recent_tool_evidence() {
|
||||
call_id: "call-12".to_string(),
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
});
|
||||
let next_transcript = TranscriptConfig::default().build(&items);
|
||||
let next_transcript = TranscriptConfig::default().build(&items).entries;
|
||||
let next_first_retained_tool = next_transcript
|
||||
.iter()
|
||||
.find(|entry| entry.contains("tool exec_command call:"))
|
||||
@@ -329,7 +329,8 @@ fn transcript_preserves_newest_manual_approval_when_message_budget_overflows() {
|
||||
max_message_transcript_tokens: message_budget,
|
||||
..TranscriptConfig::default()
|
||||
}
|
||||
.build(&items);
|
||||
.build(&items)
|
||||
.entries;
|
||||
|
||||
assert_eq!(transcript, vec![approval_entry]);
|
||||
}
|
||||
@@ -377,7 +378,8 @@ fn rejected_message_does_not_evict_retained_tool_entries() {
|
||||
max_message_transcript_tokens: TruncationPolicy::Bytes(user_entry.len()).token_budget(),
|
||||
..TranscriptConfig::default()
|
||||
}
|
||||
.build(&items);
|
||||
.build(&items)
|
||||
.entries;
|
||||
|
||||
assert_eq!(transcript, expected);
|
||||
}
|
||||
@@ -409,7 +411,7 @@ fn transcript_evicts_non_user_entries_in_cacheable_chunks() {
|
||||
internal_chat_message_metadata_passthrough: None,
|
||||
}),
|
||||
);
|
||||
config.build(&items)
|
||||
config.build(&items).entries
|
||||
};
|
||||
|
||||
let first_overflow = build_transcript(5);
|
||||
@@ -469,7 +471,7 @@ fn transcript_truncates_tool_results_using_standard_budget() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
let result = transcript
|
||||
.iter()
|
||||
.find(|entry| entry.contains(" result: "))
|
||||
@@ -497,7 +499,7 @@ fn transcript_preserves_named_unpaired_tool_source() {
|
||||
}];
|
||||
|
||||
assert_eq!(
|
||||
TranscriptConfig::default().build(&items),
|
||||
TranscriptConfig::default().build(&items).entries,
|
||||
vec!["[1] tool slack.notifications result: new message\n"]
|
||||
);
|
||||
|
||||
@@ -510,7 +512,7 @@ fn transcript_preserves_named_unpaired_tool_source() {
|
||||
]);
|
||||
}
|
||||
assert_eq!(
|
||||
TranscriptConfig::default().build(&items),
|
||||
TranscriptConfig::default().build(&items).entries,
|
||||
vec!["[1] tool slack.notifications result: [non-text output]\n"]
|
||||
);
|
||||
}
|
||||
@@ -542,7 +544,8 @@ fn configured_reasoning_counts_against_message_budget() {
|
||||
sources: vec![TranscriptSource::Reasoning],
|
||||
..TranscriptConfig::default()
|
||||
}
|
||||
.build(&items);
|
||||
.build(&items)
|
||||
.entries;
|
||||
|
||||
assert!(transcript[0].contains("user turn 0:"));
|
||||
assert!(
|
||||
@@ -598,7 +601,7 @@ fn transcript_keeps_only_manual_approval_developer_messages() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![format!("[1] developer: {approval_text}\n")]
|
||||
@@ -672,7 +675,7 @@ fn transcript_omits_media_payloads_and_keeps_readable_content() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![
|
||||
@@ -728,7 +731,7 @@ fn transcript_omits_encrypted_messages_arguments_and_tool_outputs() {
|
||||
},
|
||||
];
|
||||
|
||||
let transcript = TranscriptConfig::default().build(&items);
|
||||
let transcript = TranscriptConfig::default().build(&items).entries;
|
||||
assert_eq!(
|
||||
transcript,
|
||||
vec![
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use codex_extension_api::ExtensionMetrics;
|
||||
|
||||
pub(crate) const CLASSIFICATION_TRUNCATION_METRIC: &str =
|
||||
"codex.guardian_v2.classification.truncation";
|
||||
pub(crate) const CLASSIFICATION_TRUNCATION_BYTES_METRIC: &str =
|
||||
"codex.guardian_v2.classification.truncation.bytes";
|
||||
|
||||
pub(crate) struct TruncationObservation {
|
||||
pub(crate) component: &'static str,
|
||||
pub(crate) original_bytes: usize,
|
||||
pub(crate) retained_bytes: usize,
|
||||
}
|
||||
|
||||
/// Aggregates actual context reductions once per component and classifier request.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ClassificationTruncations {
|
||||
totals: BTreeMap<(&'static str, &'static str), (usize, usize)>,
|
||||
}
|
||||
|
||||
impl ClassificationTruncations {
|
||||
pub(crate) fn record(
|
||||
&mut self,
|
||||
component: &'static str,
|
||||
original_bytes: usize,
|
||||
retained_bytes: usize,
|
||||
) {
|
||||
if original_bytes <= retained_bytes {
|
||||
return;
|
||||
}
|
||||
|
||||
let disposition = if retained_bytes == 0 {
|
||||
"omitted"
|
||||
} else {
|
||||
"truncated"
|
||||
};
|
||||
let totals = self.totals.entry((component, disposition)).or_default();
|
||||
totals.0 = totals.0.saturating_add(original_bytes);
|
||||
totals.1 = totals.1.saturating_add(retained_bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn emit(&self, metrics: Option<&dyn ExtensionMetrics>) {
|
||||
let Some(metrics) = metrics else {
|
||||
return;
|
||||
};
|
||||
|
||||
for (&(component, disposition), &(original_bytes, retained_bytes)) in &self.totals {
|
||||
let tags = [("component", component), ("disposition", disposition)];
|
||||
metrics.counter(CLASSIFICATION_TRUNCATION_METRIC, /*inc*/ 1, &tags);
|
||||
|
||||
for (measurement, bytes) in [
|
||||
("original", original_bytes),
|
||||
("retained", retained_bytes),
|
||||
("omitted", original_bytes.saturating_sub(retained_bytes)),
|
||||
] {
|
||||
let tags = [
|
||||
("component", component),
|
||||
("disposition", disposition),
|
||||
("measurement", measurement),
|
||||
];
|
||||
metrics.histogram(
|
||||
CLASSIFICATION_TRUNCATION_BYTES_METRIC,
|
||||
i64::try_from(bytes).unwrap_or(i64::MAX),
|
||||
&tags,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Extend<TruncationObservation> for ClassificationTruncations {
|
||||
fn extend<T: IntoIterator<Item = TruncationObservation>>(&mut self, observations: T) {
|
||||
for observation in observations {
|
||||
self.record(
|
||||
observation.component,
|
||||
observation.original_bytes,
|
||||
observation.retained_bytes,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user