mirror of
https://github.com/openai/codex.git
synced 2026-11-15 21:50:46 +00:00
Preserve output boundaries through token truncation (#57584)
## What changed Add `OutputBoundary` and token-budget formatting APIs that map item indices and UTF-8 text offsets to their positions after truncation. Normalize input positions, remap boundaries within omitted text, and keep end boundaries before generated omission notices. Unify code mode output formatting: merge text-only output for truncation with a warning, and budget mixed text and audio in input order while preserving images and encrypted content. Accept owned output items to reuse allocations and move retained media without cloning. ## Testing Add coverage for boundary normalization and remapping, UTF-8 offsets, zero and nonzero budgets, owned and borrowed output equivalence, and allocation reuse. GitOrigin-RevId: 598407caa88b5a1f84130c4bc2273b85ab02f7f2
This commit is contained in:
@@ -46,8 +46,7 @@ use codex_protocol::openai_models::ToolMode;
|
||||
use codex_tools::ToolName;
|
||||
use codex_utils_audio::estimate_audio_token_count;
|
||||
use codex_utils_output_truncation::TruncationPolicy;
|
||||
use codex_utils_output_truncation::formatted_truncate_text_content_items_with_policy;
|
||||
use codex_utils_output_truncation::truncate_function_output_items_with_policy;
|
||||
use codex_utils_output_truncation::formatted_truncate_function_output_items_with_policy;
|
||||
|
||||
use delegate::CodeModeCellDelegate;
|
||||
use delegate::CodeModeDispatchBroker;
|
||||
@@ -321,16 +320,7 @@ fn truncate_code_mode_result(
|
||||
) -> Vec<FunctionCallOutputContentItem> {
|
||||
let max_output_tokens = resolve_max_tokens(max_output_tokens);
|
||||
let policy = TruncationPolicy::Tokens(max_output_tokens);
|
||||
if items
|
||||
.iter()
|
||||
.all(|item| matches!(item, FunctionCallOutputContentItem::InputText { .. }))
|
||||
{
|
||||
let (truncated_items, _) =
|
||||
formatted_truncate_text_content_items_with_policy(&items, policy);
|
||||
return truncated_items;
|
||||
}
|
||||
|
||||
truncate_function_output_items_with_policy(&items, policy, estimate_audio_token_count)
|
||||
formatted_truncate_function_output_items_with_policy(items, policy, estimate_audio_token_count)
|
||||
}
|
||||
|
||||
// Submit synchronously so the recorder sees the call before the cell's dispatch gate closes.
|
||||
|
||||
Reference in New Issue
Block a user