mirror of
https://github.com/openai/codex.git
synced 2026-10-30 19:18:49 +00:00
felixxia-oai
4878401e8f
## What changed - Add the `codex-guardian-context` crate for assembling structured context shared by synchronous Guardian review and asynchronous scoring. - Preserve transcript entry roles and original byte counts, and let section contributors declare shared or consumer-specific scope. - Collect applicable sections in registration order, skip optional sections, and fail the collection when required evidence is missing. - Provide UTF-8-safe prefix/suffix truncation with approximate omitted-token accounting. ## Testing - Cover scoped registry collection, ordering, optional sections, required-evidence failures, truncation markers, and UTF-8 boundaries. GitOrigin-RevId: 19d8458403c470c9b413992dbff925a12259c3c5
30 lines
1.1 KiB
Rust
30 lines
1.1 KiB
Rust
//! Regression coverage for Guardian's truncation marker and UTF-8 boundaries.
|
|
|
|
use codex_protocol::protocol::TruncationPolicy;
|
|
use pretty_assertions::assert_eq;
|
|
|
|
use super::truncate_text;
|
|
|
|
#[test]
|
|
fn truncation_preserves_prefix_suffix_and_utf8_boundaries() {
|
|
let text = format!("start {} end", "é🙂".repeat(/*n*/ 2_000));
|
|
let truncated = truncate_text(&text, /*max_tokens*/ 200);
|
|
let omitted_tokens = TruncationPolicy::Bytes(text.len() - 800).token_budget();
|
|
let marker = format!("<truncated omitted_approx_tokens=\"{omitted_tokens}\" />");
|
|
assert!(truncated.starts_with("start "));
|
|
assert!(truncated.ends_with(" end"));
|
|
assert!(truncated.contains(&marker));
|
|
assert!(truncated.len() <= 800);
|
|
|
|
assert_eq!(truncate_text("é🙂", /*max_tokens*/ 2), "é🙂");
|
|
assert_eq!(truncate_text("", /*max_tokens*/ 0), "");
|
|
assert_eq!(
|
|
truncate_text("é🙂", /*max_tokens*/ 0),
|
|
"<truncated omitted_approx_tokens=\"2\" />"
|
|
);
|
|
assert_eq!(
|
|
truncate_text("é🙂", /*max_tokens*/ 1),
|
|
"<truncated omitted_approx_tokens=\"1\" />"
|
|
);
|
|
}
|