mirror of
https://github.com/openai/codex.git
synced 2026-10-30 19:18:49 +00:00
Share the Guardian context delivery token limit (#58071)
Expose `MAX_TEXT_PART_ESTIMATED_TOKENS` from `codex-guardian-context` and use it for native tool calls, transcript entries, and delivery-size assertions. Keep the existing 9,000-token limit while replacing duplicated limits in code and tests. GitOrigin-RevId: 90cb98eea9bade34f18db0d1666ed7ed1e93febe
This commit is contained in:
@@ -10,6 +10,7 @@ use codex_core::config::RolloutBudgetConfig;
|
||||
use codex_core::context::GuardianContextMode;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_features::Feature;
|
||||
use codex_guardian_context::MAX_TEXT_PART_ESTIMATED_TOKENS;
|
||||
use codex_history::RolloutItem;
|
||||
use codex_protocol::config_types::ApprovalsReviewer;
|
||||
use codex_protocol::models::ContentItem;
|
||||
@@ -19,6 +20,7 @@ use codex_protocol::protocol::AskForApproval;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
use codex_protocol::protocol::Op;
|
||||
use codex_protocol::protocol::ReviewDecision;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use core_test_support::ThreadIdle;
|
||||
use core_test_support::responses;
|
||||
@@ -817,7 +819,8 @@ async fn oversized_action_preserves_review_policy_and_next_review(
|
||||
let parts = reviews[0].message_input_texts("user");
|
||||
let context = parts.concat();
|
||||
assert!(context.contains(&oversized_command));
|
||||
assert!(parts.iter().all(|part| part.len() <= 36_000));
|
||||
assert!(parts.iter().all(|part| part.len()
|
||||
<= TruncationPolicy::Tokens(MAX_TEXT_PART_ESTIMATED_TOKENS).byte_budget()));
|
||||
let transcript = context
|
||||
.split_once(">>> TRANSCRIPT START\n")
|
||||
.expect("full transcript")
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
use crate::ContextSection;
|
||||
use crate::GuardianToolDescriptions;
|
||||
use crate::MAX_TEXT_PART_ESTIMATED_TOKENS;
|
||||
use crate::SectionContributor;
|
||||
use crate::SectionError;
|
||||
use crate::SectionInput;
|
||||
@@ -181,7 +182,7 @@ impl SectionContributor for PlannedActionSection {
|
||||
});
|
||||
}
|
||||
if let PlannedActionKind::ToolCall(call) = &action.kind
|
||||
&& crate::native_tool_call::estimate_tokens(call) > 9_000
|
||||
&& crate::native_tool_call::estimate_tokens(call) > MAX_TEXT_PART_ESTIMATED_TOKENS
|
||||
{
|
||||
return Err(SectionError::EvidenceLimitExceeded {
|
||||
section: "native_tool_call",
|
||||
|
||||
@@ -648,9 +648,9 @@ fn delivered_message(
|
||||
}
|
||||
}
|
||||
|
||||
// Preserve the delivery cap below the 10K-token context-item guidance. The
|
||||
// estimate uses four bytes per token, so each UTF-8-safe part is at most 36KB.
|
||||
const MAX_TEXT_PART_ESTIMATED_TOKENS: usize = 9_000;
|
||||
/// Delivery cap below the 10K-token context-item guidance, also applied to native calls.
|
||||
/// The estimate uses four bytes per token, so each UTF-8-safe text part is at most 36KB.
|
||||
pub const MAX_TEXT_PART_ESTIMATED_TOKENS: usize = 9_000;
|
||||
|
||||
/// Bounds individual text parts without changing source text or entry identity.
|
||||
/// Budget estimates include the extra wire framing before transport conversion.
|
||||
|
||||
@@ -103,9 +103,8 @@ fn long_text_and_file_image_delivery_is_lossless_bounded_and_fully_budgeted() {
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(parts.concat(), text);
|
||||
assert!(
|
||||
parts
|
||||
.iter()
|
||||
.all(|part| part.len() <= TruncationPolicy::Tokens(9_000).byte_budget())
|
||||
parts.iter().all(|part| part.len()
|
||||
<= TruncationPolicy::Tokens(MAX_TEXT_PART_ESTIMATED_TOKENS).byte_budget())
|
||||
);
|
||||
assert!(estimated >= crate::estimate_input_tokens(&messages[0]));
|
||||
let mut expected_inputs = parts
|
||||
|
||||
@@ -96,6 +96,7 @@ pub use codex_protocol::TranscriptFormat;
|
||||
pub use composition::CollectedContext;
|
||||
pub use composition::ComposedContext;
|
||||
pub use composition::ContextPresentation;
|
||||
pub use composition::MAX_TEXT_PART_ESTIMATED_TOKENS;
|
||||
pub use composition::PreparedTranscript;
|
||||
pub use preparation::ContextProfile;
|
||||
pub use preparation::SyncImagePreparer;
|
||||
|
||||
@@ -24,6 +24,7 @@ use codex_protocol::protocol::InterAgentCommunication;
|
||||
use crate::ContextSection;
|
||||
use crate::ConversationTranscriptEntry;
|
||||
use crate::ConversationTranscriptEntryKind;
|
||||
use crate::MAX_TEXT_PART_ESTIMATED_TOKENS;
|
||||
use crate::PlannedAction;
|
||||
use crate::PlannedActionKind;
|
||||
use crate::SectionContributor;
|
||||
@@ -322,7 +323,10 @@ fn collect_transcript_for_review(
|
||||
entries.push(ConversationTranscriptEntry {
|
||||
kind,
|
||||
content: if crate::estimate_input_tokens(item)
|
||||
<= config.entry_limits.tool_tokens.min(9_000)
|
||||
<= config
|
||||
.entry_limits
|
||||
.tool_tokens
|
||||
.min(MAX_TEXT_PART_ESTIMATED_TOKENS)
|
||||
{
|
||||
TranscriptContent::NativeMessage(Box::new(item.clone()))
|
||||
} else {
|
||||
|
||||
Reference in New Issue
Block a user