mirror of
https://github.com/openai/codex.git
synced 2026-11-13 21:30:47 +00:00
felixxia-oai
4878401e8f
## What changed - Add the `codex-guardian-context` crate for assembling structured context shared by synchronous Guardian review and asynchronous scoring. - Preserve transcript entry roles and original byte counts, and let section contributors declare shared or consumer-specific scope. - Collect applicable sections in registration order, skip optional sections, and fail the collection when required evidence is missing. - Provide UTF-8-safe prefix/suffix truncation with approximate omitted-token accounting. ## Testing - Cover scoped registry collection, ordering, optional sections, required-evidence failures, truncation markers, and UTF-8 boundaries. GitOrigin-RevId: 19d8458403c470c9b413992dbff925a12259c3c5
43 lines
1.5 KiB
Rust
43 lines
1.5 KiB
Rust
//! Guardian's shared UTF-8-safe, prefix/suffix text truncation primitive.
|
|
//!
|
|
//! The existing XML omission marker is preserved, including returning the whole
|
|
//! marker when a token budget is too small to contain it.
|
|
|
|
use codex_protocol::protocol::TruncationPolicy;
|
|
|
|
/// Truncates text using Guardian's approximate token budget and omission marker.
|
|
///
|
|
/// Retains both ends on UTF-8 boundaries. Budgets smaller than the marker still
|
|
/// return the marker, so callers should reserve room for that fixed overhead.
|
|
pub fn truncate_text(text: &str, max_tokens: usize) -> String {
|
|
let max_bytes = TruncationPolicy::Tokens(max_tokens).byte_budget();
|
|
if text.len() <= max_bytes {
|
|
return text.to_owned();
|
|
}
|
|
|
|
let omitted_tokens =
|
|
TruncationPolicy::Bytes(text.len().saturating_sub(max_bytes)).token_budget();
|
|
let marker = format!("<truncated omitted_approx_tokens=\"{omitted_tokens}\" />");
|
|
if max_bytes <= marker.len() {
|
|
return marker;
|
|
}
|
|
|
|
let available_bytes = max_bytes - marker.len();
|
|
let prefix_bytes = available_bytes / 2;
|
|
let suffix_bytes = available_bytes - prefix_bytes;
|
|
let mut prefix_end = prefix_bytes;
|
|
while !text.is_char_boundary(prefix_end) {
|
|
prefix_end -= 1;
|
|
}
|
|
let mut suffix_start = text.len() - suffix_bytes;
|
|
while !text.is_char_boundary(suffix_start) {
|
|
suffix_start += 1;
|
|
}
|
|
|
|
format!("{}{marker}{}", &text[..prefix_end], &text[suffix_start..])
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "truncation_tests.rs"]
|
|
mod tests;
|