mirror of
https://github.com/openai/codex.git
synced 2026-10-29 19:08:43 +00:00
Preserve output boundaries through token truncation (#57584)
## What changed Add `OutputBoundary` and token-budget formatting APIs that map item indices and UTF-8 text offsets to their positions after truncation. Normalize input positions, remap boundaries within omitted text, and keep end boundaries before generated omission notices. Unify code mode output formatting: merge text-only output for truncation with a warning, and budget mixed text and audio in input order while preserving images and encrypted content. Accept owned output items to reuse allocations and move retained media without cloning. ## Testing Add coverage for boundary normalization and remapping, UTF-8 offsets, zero and nonzero budgets, owned and borrowed output equivalence, and allocation reuse. GitOrigin-RevId: 598407caa88b5a1f84130c4bc2273b85ab02f7f2
This commit is contained in:
@@ -12,6 +12,17 @@ pub struct TextTruncation {
|
||||
pub marker: String,
|
||||
}
|
||||
|
||||
impl TextTruncation {
|
||||
/// Applies this edit to the string it was planned from.
|
||||
pub fn apply(&self, original: &str) -> String {
|
||||
assemble_truncated_output(
|
||||
&original[..self.removed.start],
|
||||
&original[self.removed.end..],
|
||||
&self.marker,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Plans the existing head/tail token truncation. The marker is outside the
|
||||
/// retained-text budget, just as it is in [`truncate_middle_with_token_budget`].
|
||||
pub fn plan_token_truncation(s: &str, max_tokens: usize) -> Option<TextTruncation> {
|
||||
@@ -70,11 +81,7 @@ pub fn truncate_middle_with_token_budget(s: &str, max_tokens: usize) -> (String,
|
||||
let Some(truncation) = plan_token_truncation(s, max_tokens) else {
|
||||
return (s.to_string(), None);
|
||||
};
|
||||
let truncated = assemble_truncated_output(
|
||||
&s[..truncation.removed.start],
|
||||
&s[truncation.removed.end..],
|
||||
&truncation.marker,
|
||||
);
|
||||
let truncated = truncation.apply(s);
|
||||
let total_tokens = u64::try_from(approx_token_count(s)).unwrap_or(u64::MAX);
|
||||
|
||||
if truncated == s {
|
||||
|
||||
Reference in New Issue
Block a user