mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
## What changed Preserve complete tool-call text when collecting synchronous Guardian context, then apply the profile's tool token limit when preparing the transcript. Calculate retained bytes and token usage from the truncated text. ## Testing Extend the oversized-action regression test to verify that a subsequent review includes the previous call's arguments truncated to the 1,000-token history limit. GitOrigin-RevId: 2b52505d9ccfa3651b3535e67cc12597be819459
327 lines
14 KiB
Rust
327 lines
14 KiB
Rust
//! Resolved Guardian evidence profiles, transcript admission and retention.
|
|
//! Sync keeps recent entries; async protects approvals/final answers and evicts
|
|
//! in cacheable chunks. Hosts still own history snapshots and delivery cursors.
|
|
//! User messages and manual approvals stay complete until whole-request admission.
|
|
//! They may be shortened with explicit markers only during budget recovery.
|
|
//! The host selects line or JSON rendering; JSON keeps embedded headers inside text.
|
|
|
|
use codex_protocol::protocol::TruncationPolicy;
|
|
|
|
use crate::BudgetPriority;
|
|
use crate::Budgeted;
|
|
use crate::ContextTarget;
|
|
use crate::ConversationTranscriptConfig;
|
|
use crate::ConversationTranscriptEntry;
|
|
use crate::ConversationTranscriptEntryKind;
|
|
use crate::ConversationTranscriptOptions;
|
|
use crate::PreparedTranscript;
|
|
use crate::Retention;
|
|
use crate::TranscriptContent;
|
|
use crate::TranscriptEntryLimits;
|
|
use crate::TranscriptFormat;
|
|
use crate::TranscriptRetentionConfig;
|
|
use crate::TruncationObservation;
|
|
use crate::transcript_record::TranscriptRecord;
|
|
|
|
use self::window::TranscriptWindow;
|
|
mod window;
|
|
|
|
const MIN_RECENT_TOOL_ENTRIES: usize = 5;
|
|
|
|
/// Request-local policy resolved from the consumer's model and configuration.
|
|
/// Registry scope follows `target`; source flags and caps apply before retention.
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
pub struct ContextProfile {
|
|
pub target: ContextTarget,
|
|
pub transcript_format: TranscriptFormat,
|
|
pub transcript: ConversationTranscriptConfig,
|
|
pub retention: TranscriptRetentionConfig,
|
|
pub include_images: bool,
|
|
}
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
enum TranscriptEntryKind {
|
|
User,
|
|
ManualApproval,
|
|
ProtectedMessage,
|
|
Message,
|
|
Tool,
|
|
}
|
|
|
|
struct TranscriptEntry {
|
|
kind: TranscriptEntryKind,
|
|
content: TranscriptContent,
|
|
tokens: usize,
|
|
original_bytes: usize,
|
|
retained_bytes: usize,
|
|
source: Option<codex_history::RetainedSource>,
|
|
}
|
|
|
|
impl ContextProfile {
|
|
/// Existing synchronous defaults before the host resolves the REPL output cap.
|
|
pub const fn synchronous() -> Self {
|
|
Self {
|
|
target: ContextTarget::Sync,
|
|
transcript_format: TranscriptFormat::Line,
|
|
transcript: ConversationTranscriptConfig {
|
|
options: ConversationTranscriptOptions {
|
|
include_tool_calls: true,
|
|
include_tool_outputs: true,
|
|
include_reasoning: false,
|
|
},
|
|
entry_limits: TranscriptEntryLimits {
|
|
message_tokens: 5_000,
|
|
tool_tokens: 1_000,
|
|
node_repl_output_tokens: 1_000,
|
|
},
|
|
},
|
|
retention: TranscriptRetentionConfig {
|
|
max_message_transcript_tokens: 20_000,
|
|
max_tool_transcript_tokens: 10_000,
|
|
max_recent_non_user_entries: 40,
|
|
},
|
|
include_images: false,
|
|
}
|
|
}
|
|
|
|
/// Existing asynchronous defaults before model and local overrides.
|
|
pub const fn asynchronous() -> Self {
|
|
Self {
|
|
target: ContextTarget::Async,
|
|
transcript_format: TranscriptFormat::Line,
|
|
transcript: ConversationTranscriptConfig {
|
|
options: ConversationTranscriptOptions {
|
|
include_tool_calls: true,
|
|
include_tool_outputs: true,
|
|
include_reasoning: false,
|
|
},
|
|
entry_limits: TranscriptEntryLimits {
|
|
message_tokens: 2_000,
|
|
tool_tokens: 1_000,
|
|
node_repl_output_tokens: 1_000,
|
|
},
|
|
},
|
|
retention: TranscriptRetentionConfig {
|
|
max_message_transcript_tokens: 10_000,
|
|
max_tool_transcript_tokens: 10_000,
|
|
max_recent_non_user_entries: 40,
|
|
},
|
|
include_images: true,
|
|
}
|
|
}
|
|
|
|
/// Prepares structured, bounded entries without advancing the host's full/delta cursor.
|
|
/// The host supplies the slice and original offset; empty placeholders depend
|
|
/// on its full/delta presentation and are supplied after selection.
|
|
pub fn prepare_transcript(
|
|
&self,
|
|
transcript_entries: &[ConversationTranscriptEntry],
|
|
entry_number_offset: usize,
|
|
) -> PreparedTranscript {
|
|
let entries = transcript_entries
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(index, entry)| {
|
|
let kind = match &entry.kind {
|
|
ConversationTranscriptEntryKind::User => TranscriptEntryKind::User,
|
|
ConversationTranscriptEntryKind::Developer => {
|
|
TranscriptEntryKind::ManualApproval
|
|
}
|
|
ConversationTranscriptEntryKind::ProtectedAssistant => {
|
|
TranscriptEntryKind::ProtectedMessage
|
|
}
|
|
ConversationTranscriptEntryKind::Assistant
|
|
| ConversationTranscriptEntryKind::Reasoning => TranscriptEntryKind::Message,
|
|
ConversationTranscriptEntryKind::ToolCall(_)
|
|
| ConversationTranscriptEntryKind::ToolOutput(_)
|
|
| ConversationTranscriptEntryKind::NodeReplToolOutput(_) => {
|
|
TranscriptEntryKind::Tool
|
|
}
|
|
};
|
|
let number = index + entry_number_offset + 1;
|
|
let suffix = match self.target {
|
|
ContextTarget::Sync => "",
|
|
ContextTarget::Async => "\n",
|
|
};
|
|
let retained_source = entry
|
|
.retained_source
|
|
.as_ref()
|
|
.filter(|_| entry.kind == ConversationTranscriptEntryKind::User);
|
|
let prepare_text = |text: &str| {
|
|
let text = match self.target {
|
|
ContextTarget::Sync => match &entry.kind {
|
|
ConversationTranscriptEntryKind::ToolCall(_) => {
|
|
std::borrow::Cow::Owned(crate::truncate_text(
|
|
text,
|
|
self.transcript.entry_limits.tool_tokens,
|
|
))
|
|
}
|
|
ConversationTranscriptEntryKind::User
|
|
| ConversationTranscriptEntryKind::Developer
|
|
| ConversationTranscriptEntryKind::Assistant
|
|
| ConversationTranscriptEntryKind::ProtectedAssistant
|
|
| ConversationTranscriptEntryKind::Reasoning
|
|
| ConversationTranscriptEntryKind::ToolOutput(_)
|
|
| ConversationTranscriptEntryKind::NodeReplToolOutput(_) => {
|
|
std::borrow::Cow::Borrowed(text)
|
|
}
|
|
},
|
|
ContextTarget::Async => std::borrow::Cow::Borrowed(text),
|
|
};
|
|
let retained_bytes = text.len();
|
|
let record = TranscriptRecord::new(
|
|
&entry.kind,
|
|
number,
|
|
text.into_owned(),
|
|
retained_source.map(|retained| retained.order.clone()),
|
|
self.transcript_format,
|
|
suffix,
|
|
);
|
|
let tokens =
|
|
TruncationPolicy::Bytes(record.rendered().text_bytes).token_budget();
|
|
(TranscriptContent::Record(record), tokens, retained_bytes)
|
|
};
|
|
let (content, tokens, retained_bytes) = match &entry.content {
|
|
TranscriptContent::Text(text) => prepare_text(text),
|
|
TranscriptContent::Record(record) => prepare_text(record.text()),
|
|
TranscriptContent::AgentMessage(message) => (
|
|
TranscriptContent::AgentMessage(message.clone()),
|
|
crate::estimate_input_tokens(message),
|
|
entry.original_bytes,
|
|
),
|
|
};
|
|
TranscriptEntry {
|
|
kind,
|
|
tokens,
|
|
content,
|
|
original_bytes: entry.original_bytes,
|
|
retained_bytes,
|
|
source: retained_source.map(|retained| retained.source.clone()),
|
|
}
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let mut included = vec![false; entries.len()];
|
|
let mut authorization_tokens = 0usize;
|
|
for (index, entry) in entries.iter().enumerate() {
|
|
if matches!(
|
|
entry.kind,
|
|
TranscriptEntryKind::User | TranscriptEntryKind::ManualApproval
|
|
) {
|
|
included[index] = true;
|
|
authorization_tokens = authorization_tokens.saturating_add(entry.tokens);
|
|
}
|
|
}
|
|
match self.target {
|
|
ContextTarget::Sync => {
|
|
let mut message_tokens = authorization_tokens;
|
|
let mut tool_tokens = 0usize;
|
|
let mut retained_non_user_entries = 0;
|
|
for (index, entry) in entries.iter().enumerate().rev() {
|
|
if matches!(
|
|
entry.kind,
|
|
TranscriptEntryKind::User | TranscriptEntryKind::ManualApproval
|
|
) || retained_non_user_entries >= self.retention.max_recent_non_user_entries
|
|
{
|
|
continue;
|
|
}
|
|
let (tokens, limit) = if entry.kind == TranscriptEntryKind::Tool {
|
|
(&mut tool_tokens, self.retention.max_tool_transcript_tokens)
|
|
} else {
|
|
(
|
|
&mut message_tokens,
|
|
self.retention.max_message_transcript_tokens,
|
|
)
|
|
};
|
|
if tokens.saturating_add(entry.tokens) <= limit {
|
|
included[index] = true;
|
|
retained_non_user_entries += 1;
|
|
*tokens += entry.tokens;
|
|
}
|
|
}
|
|
}
|
|
ContextTarget::Async => {
|
|
let available = self
|
|
.retention
|
|
.max_message_transcript_tokens
|
|
.saturating_sub(authorization_tokens);
|
|
let mut window = TranscriptWindow::new(&entries, &self.retention, available);
|
|
for index in 0..entries.len() {
|
|
window.insert(index);
|
|
}
|
|
for index in window.into_indices() {
|
|
included[index] = true;
|
|
}
|
|
}
|
|
}
|
|
let omission_note = (self.target == ContextTarget::Sync
|
|
&& included.iter().any(|included| !included))
|
|
.then(|| crate::transcript::TRANSCRIPT_OMISSION_NOTICE.to_owned());
|
|
let mut truncations = Vec::new();
|
|
let mut items: Vec<_> = entries
|
|
.into_iter()
|
|
.enumerate()
|
|
.filter_map(|(index, entry)| {
|
|
let component = match entry.kind {
|
|
TranscriptEntryKind::User => "transcript_user",
|
|
TranscriptEntryKind::ManualApproval
|
|
| TranscriptEntryKind::ProtectedMessage
|
|
| TranscriptEntryKind::Message => "transcript_message",
|
|
TranscriptEntryKind::Tool => "transcript_tool",
|
|
};
|
|
let retained_bytes = if included[index] {
|
|
entry.retained_bytes
|
|
} else {
|
|
0
|
|
};
|
|
if self.target == ContextTarget::Async && entry.original_bytes > retained_bytes {
|
|
truncations.push(TruncationObservation {
|
|
component,
|
|
original_bytes: entry.original_bytes,
|
|
retained_bytes,
|
|
});
|
|
}
|
|
if included[index] {
|
|
Some(match entry.kind {
|
|
TranscriptEntryKind::User | TranscriptEntryKind::ManualApproval => {
|
|
if let Some(source) = entry.source {
|
|
let mut item = Budgeted::required(entry.content);
|
|
item.source = Some(source);
|
|
item
|
|
} else {
|
|
Budgeted::historical(entry.content)
|
|
}
|
|
}
|
|
TranscriptEntryKind::ProtectedMessage => Budgeted::required(entry.content),
|
|
TranscriptEntryKind::Message => {
|
|
Budgeted::optional(entry.content, BudgetPriority::Commentary)
|
|
}
|
|
TranscriptEntryKind::Tool => {
|
|
Budgeted::optional(entry.content, BudgetPriority::Tool)
|
|
}
|
|
})
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.collect();
|
|
// Keep the newest tool evidence even when the aggregate allowance is tight.
|
|
for item in items
|
|
.iter_mut()
|
|
.rev()
|
|
.filter(|item| item.retention == Retention::Optional(BudgetPriority::Tool))
|
|
.take(MIN_RECENT_TOOL_ENTRIES)
|
|
{
|
|
item.retention = Retention::Required;
|
|
}
|
|
PreparedTranscript {
|
|
items,
|
|
omission_note,
|
|
truncations,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "profile_tests.rs"]
|
|
mod tests;
|