mirror of
https://github.com/openai/codex.git
synced 2026-10-20 17:38:45 +00:00
## Why Skill aliases already identify their source, but `skills.read` searched the cloud catalog before the executor catalog even for executor aliases. ## What changed Parse package locators to recognize cloud (`c0/...`), executor (`e0/...`), and host (`r0/...`) aliases. Restrict cloud and executor alias lookups to the matching catalog, while retaining the existing search order for other package strings. Share the source-to-prefix mapping between alias generation and parsing, and preserve the string representation and tool schema for package locators. ## Testing Extend alias tests to cover source recognition, JSON round trips, and malformed alias strings, including leading zeros, missing suffixes, non-ASCII prefixes, and overflowing indices. GitOrigin-RevId: f835e47a244aa0cf9eb95980ea23f4d7fcedd511
1116 lines
36 KiB
Rust
1116 lines
36 KiB
Rust
use std::borrow::Cow;
|
|
use std::num::NonZeroUsize;
|
|
|
|
use codex_protocol::protocol::SkillScope;
|
|
use codex_utils_string::approx_token_count;
|
|
use codex_utils_string::take_bytes_at_char_boundary;
|
|
use serde::Deserialize;
|
|
use serde::Serialize;
|
|
|
|
use crate::aliases::AliasPlan;
|
|
use crate::catalog::SkillCatalog;
|
|
use crate::catalog::SkillCatalogEntry;
|
|
use crate::catalog::SkillSourceKind;
|
|
use crate::catalog_prompt::SkillPromptKind;
|
|
use crate::catalog_prompt::render_available_skills_body;
|
|
use crate::fragments::AvailableSkillsInstructions;
|
|
use crate::host_aliases::shared_host_alias_roots;
|
|
|
|
const DEFAULT_SKILL_METADATA_CHAR_BUDGET: usize = 8_000;
|
|
const MAX_CONFIGURED_SKILL_METADATA_TOKEN_BUDGET: usize = 10_000;
|
|
const MAX_SKILL_PROMPT_BYTES: usize = 8_000;
|
|
const SKILL_METADATA_CONTEXT_WINDOW_PERCENT: usize = 2;
|
|
const MAX_CATALOG_SKILL_DESCRIPTION_CHARS: usize = 1_024;
|
|
const TRUNCATED_SKILL_DESCRIPTION_SUFFIX: &str = "...";
|
|
const APPROX_BYTES_PER_TOKEN: usize = 4;
|
|
const SKILL_DESCRIPTIONS_REMOVED_WARNING_PREFIX: &str =
|
|
"Exceeded skills context budget. All skill descriptions were removed and";
|
|
pub(crate) const MAX_SKILL_NAME_BYTES: usize = 256;
|
|
pub(crate) const MAX_SKILL_PATH_BYTES: usize = 1_024;
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub(crate) enum SkillCatalogRenderPolicy {
|
|
CoreCompatible,
|
|
ExtensionCompatible,
|
|
}
|
|
|
|
impl SkillCatalogRenderPolicy {
|
|
fn description(self, entry: &SkillCatalogEntry) -> &str {
|
|
match self {
|
|
Self::CoreCompatible => entry.description.as_str(),
|
|
Self::ExtensionCompatible => entry
|
|
.short_description
|
|
.as_deref()
|
|
.unwrap_or(entry.description.as_str()),
|
|
}
|
|
}
|
|
|
|
fn order_entries(self, entries: &mut [&SkillCatalogEntry]) {
|
|
match self {
|
|
Self::CoreCompatible => {
|
|
let scope_rank = |entry: &SkillCatalogEntry| match entry.prompt_scope() {
|
|
Some(SkillScope::System) => 0,
|
|
Some(SkillScope::Admin) => 1,
|
|
Some(SkillScope::Repo) => 2,
|
|
Some(SkillScope::User) => 3,
|
|
None => 4,
|
|
};
|
|
entries.sort_by(|a, b| {
|
|
scope_rank(a)
|
|
.cmp(&scope_rank(b))
|
|
.then_with(|| a.name.cmp(&b.name))
|
|
.then_with(|| a.main_prompt.as_str().cmp(b.main_prompt.as_str()))
|
|
});
|
|
}
|
|
Self::ExtensionCompatible => {}
|
|
}
|
|
}
|
|
|
|
fn includes_omission_notice(self) -> bool {
|
|
match self {
|
|
Self::CoreCompatible => false,
|
|
Self::ExtensionCompatible => true,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub(crate) enum SkillMetadataBudget {
|
|
Tokens(usize),
|
|
Characters(usize),
|
|
}
|
|
|
|
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
|
pub(crate) struct SkillRenderReport {
|
|
pub(crate) total_count: usize,
|
|
pub(crate) included_count: usize,
|
|
pub(crate) omitted_count: usize,
|
|
pub(crate) truncated_description_chars: usize,
|
|
pub(crate) truncated_description_count: usize,
|
|
}
|
|
|
|
impl SkillRenderReport {
|
|
pub(crate) fn warning_message(&self) -> Option<String> {
|
|
if self.omitted_count > 0 {
|
|
let skill_word = if self.omitted_count == 1 {
|
|
"skill"
|
|
} else {
|
|
"skills"
|
|
};
|
|
let verb = if self.omitted_count == 1 {
|
|
"was"
|
|
} else {
|
|
"were"
|
|
};
|
|
return Some(format!(
|
|
"{} {} additional {} {} not included in the model-visible skills list.",
|
|
SKILL_DESCRIPTIONS_REMOVED_WARNING_PREFIX, self.omitted_count, skill_word, verb
|
|
));
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
pub(crate) fn average_truncated_description_chars(&self) -> usize {
|
|
if self.total_count == 0 || self.truncated_description_chars == 0 {
|
|
return 0;
|
|
}
|
|
|
|
self.truncated_description_chars
|
|
.saturating_add(self.total_count.saturating_sub(1))
|
|
/ self.total_count
|
|
}
|
|
}
|
|
|
|
pub(crate) fn skill_metadata_budget(
|
|
context_window: Option<i64>,
|
|
max_context_tokens: Option<NonZeroUsize>,
|
|
) -> SkillMetadataBudget {
|
|
if let Some(max_context_tokens) = max_context_tokens {
|
|
return SkillMetadataBudget::Tokens(
|
|
max_context_tokens
|
|
.get()
|
|
.min(MAX_CONFIGURED_SKILL_METADATA_TOKEN_BUDGET),
|
|
);
|
|
}
|
|
|
|
context_window
|
|
.and_then(|window| usize::try_from(window).ok())
|
|
.filter(|window| *window > 0)
|
|
.map(|window| {
|
|
SkillMetadataBudget::Tokens(
|
|
window
|
|
.saturating_mul(SKILL_METADATA_CONTEXT_WINDOW_PERCENT)
|
|
.saturating_div(100)
|
|
.max(1),
|
|
)
|
|
})
|
|
.unwrap_or(SkillMetadataBudget::Characters(
|
|
DEFAULT_SKILL_METADATA_CHAR_BUDGET,
|
|
))
|
|
}
|
|
|
|
fn metadata_line_cost(budget: SkillMetadataBudget, line: &str) -> usize {
|
|
let line = format!("{line}\n");
|
|
match budget {
|
|
SkillMetadataBudget::Tokens(_) => approx_token_count(&line),
|
|
SkillMetadataBudget::Characters(_) => line.chars().count(),
|
|
}
|
|
}
|
|
|
|
impl SkillMetadataBudget {
|
|
pub(crate) fn with_limit(self, limit: usize) -> Self {
|
|
match self {
|
|
Self::Tokens(_) => Self::Tokens(limit),
|
|
Self::Characters(_) => Self::Characters(limit),
|
|
}
|
|
}
|
|
|
|
pub(crate) fn limit(self) -> usize {
|
|
match self {
|
|
Self::Tokens(limit) | Self::Characters(limit) => limit,
|
|
}
|
|
}
|
|
|
|
fn cost_from_counts(self, chars: usize, bytes: usize) -> usize {
|
|
match self {
|
|
Self::Tokens(_) => {
|
|
bytes.saturating_add(APPROX_BYTES_PER_TOKEN.saturating_sub(1))
|
|
/ APPROX_BYTES_PER_TOKEN
|
|
}
|
|
Self::Characters(_) => chars,
|
|
}
|
|
}
|
|
|
|
fn cost(self, text: &str) -> usize {
|
|
match self {
|
|
Self::Tokens(_) => approx_token_count(text),
|
|
Self::Characters(_) => text.chars().count(),
|
|
}
|
|
}
|
|
}
|
|
|
|
struct SkillLine<'a> {
|
|
name: &'a str,
|
|
description: Cow<'a, str>,
|
|
locator: String,
|
|
locator_kind: &'static str,
|
|
}
|
|
|
|
impl<'a> SkillLine<'a> {
|
|
fn new(entry: &'a SkillCatalogEntry, policy: SkillCatalogRenderPolicy) -> Self {
|
|
let locator = match &entry.authority.kind {
|
|
SkillSourceKind::Executor | SkillSourceKind::Cloud => entry.id.0.as_str(),
|
|
SkillSourceKind::Host | SkillSourceKind::Custom(_) => entry.rendered_path(),
|
|
};
|
|
Self::with_locator(entry, policy, locator.to_string())
|
|
}
|
|
|
|
fn with_locator(
|
|
entry: &'a SkillCatalogEntry,
|
|
policy: SkillCatalogRenderPolicy,
|
|
locator: String,
|
|
) -> Self {
|
|
let description = policy.description(entry);
|
|
Self {
|
|
name: entry.name.as_str(),
|
|
description: truncate_catalog_skill_description(description),
|
|
locator,
|
|
locator_kind: match &entry.authority.kind {
|
|
SkillSourceKind::Host => "file",
|
|
SkillSourceKind::Executor => "executor package",
|
|
SkillSourceKind::Cloud => "cloud package",
|
|
SkillSourceKind::Custom(_) => "custom resource",
|
|
},
|
|
}
|
|
}
|
|
|
|
fn full_cost(&self, budget: SkillMetadataBudget) -> usize {
|
|
metadata_line_cost(budget, &self.render_full())
|
|
}
|
|
|
|
fn minimum_cost(&self, budget: SkillMetadataBudget) -> usize {
|
|
metadata_line_cost(budget, &self.render_minimum())
|
|
}
|
|
|
|
fn description_char_count(&self) -> usize {
|
|
self.description.chars().count()
|
|
}
|
|
|
|
fn render_full(&self) -> String {
|
|
self.render_with_description(self.description.as_ref())
|
|
}
|
|
|
|
fn render_minimum(&self) -> String {
|
|
self.render_with_description("")
|
|
}
|
|
|
|
fn render_with_description_chars(&self, description_chars: usize) -> String {
|
|
let end = self
|
|
.description
|
|
.char_indices()
|
|
.nth(description_chars)
|
|
.map_or(self.description.len(), |(index, _)| index);
|
|
self.render_with_description(&self.description[..end])
|
|
}
|
|
|
|
fn render_with_description(&self, description: &str) -> String {
|
|
let name = self.name;
|
|
let locator = self.locator.as_str();
|
|
let locator_kind = self.locator_kind;
|
|
if description.is_empty() {
|
|
format!("- {name}: ({locator_kind}: {locator})")
|
|
} else {
|
|
format!("- {name}: {description} ({locator_kind}: {locator})")
|
|
}
|
|
}
|
|
}
|
|
|
|
struct RenderedSkillLine {
|
|
line: String,
|
|
}
|
|
|
|
struct RenderedSkillLines {
|
|
lines: Vec<RenderedSkillLine>,
|
|
omitted_count: usize,
|
|
truncated_description_chars: usize,
|
|
truncated_description_count: usize,
|
|
}
|
|
|
|
struct DescriptionBudgetLine {
|
|
description_char_count: usize,
|
|
extra_costs: Vec<usize>,
|
|
}
|
|
|
|
impl DescriptionBudgetLine {
|
|
fn new(line: &SkillLine<'_>, budget: SkillMetadataBudget) -> Self {
|
|
let minimum_line = line.render_minimum();
|
|
let minimum_chars = minimum_line.chars().count().saturating_add(1);
|
|
let minimum_bytes = minimum_line.len().saturating_add(1);
|
|
let minimum_cost = budget.cost_from_counts(minimum_chars, minimum_bytes);
|
|
|
|
let description_char_count = line.description.chars().count();
|
|
let mut extra_costs = Vec::with_capacity(description_char_count.saturating_add(1));
|
|
extra_costs.push(0);
|
|
|
|
let mut prefix_chars = 0usize;
|
|
let mut prefix_bytes = 0usize;
|
|
for ch in line.description.chars() {
|
|
prefix_chars = prefix_chars.saturating_add(1);
|
|
prefix_bytes = prefix_bytes.saturating_add(ch.len_utf8());
|
|
let rendered_chars = minimum_chars.saturating_add(prefix_chars).saturating_add(1);
|
|
let rendered_bytes = minimum_bytes.saturating_add(prefix_bytes).saturating_add(1);
|
|
let cost = budget
|
|
.cost_from_counts(rendered_chars, rendered_bytes)
|
|
.saturating_sub(minimum_cost);
|
|
extra_costs.push(cost);
|
|
}
|
|
|
|
Self {
|
|
description_char_count,
|
|
extra_costs,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Copy)]
|
|
enum SkillLineAllocation {
|
|
Omitted,
|
|
DescriptionChars(usize),
|
|
}
|
|
|
|
fn render_skill_lines(
|
|
skill_lines: Vec<SkillLine<'_>>,
|
|
budget: SkillMetadataBudget,
|
|
) -> RenderedSkillLines {
|
|
let allocations = allocate_skill_lines(&skill_lines, budget);
|
|
render_allocated_skill_lines(&skill_lines, &allocations)
|
|
}
|
|
|
|
fn allocate_skill_lines(
|
|
skill_lines: &[SkillLine<'_>],
|
|
budget: SkillMetadataBudget,
|
|
) -> Vec<SkillLineAllocation> {
|
|
let full_cost = skill_lines.iter().fold(0usize, |used, line| {
|
|
used.saturating_add(line.full_cost(budget))
|
|
});
|
|
if full_cost <= budget.limit() {
|
|
return skill_lines
|
|
.iter()
|
|
.map(|line| SkillLineAllocation::DescriptionChars(line.description_char_count()))
|
|
.collect();
|
|
}
|
|
|
|
let minimum_cost = skill_lines.iter().fold(0usize, |used, line| {
|
|
used.saturating_add(line.minimum_cost(budget))
|
|
});
|
|
if minimum_cost <= budget.limit() {
|
|
return allocate_description_chars(
|
|
budget,
|
|
skill_lines,
|
|
budget.limit().saturating_sub(minimum_cost),
|
|
)
|
|
.into_iter()
|
|
.map(SkillLineAllocation::DescriptionChars)
|
|
.collect();
|
|
}
|
|
|
|
let mut used = 0usize;
|
|
skill_lines
|
|
.iter()
|
|
.map(|line| {
|
|
let next_used = used.saturating_add(line.minimum_cost(budget));
|
|
if next_used <= budget.limit() {
|
|
used = next_used;
|
|
SkillLineAllocation::DescriptionChars(0)
|
|
} else {
|
|
SkillLineAllocation::Omitted
|
|
}
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
fn render_allocated_skill_lines(
|
|
skill_lines: &[SkillLine<'_>],
|
|
allocations: &[SkillLineAllocation],
|
|
) -> RenderedSkillLines {
|
|
let mut lines = Vec::new();
|
|
let mut omitted_count = 0usize;
|
|
let mut truncated_description_chars = 0usize;
|
|
let mut truncated_description_count = 0usize;
|
|
for (line, allocation) in skill_lines.iter().zip(allocations) {
|
|
let description_char_count = line.description_char_count();
|
|
match allocation {
|
|
SkillLineAllocation::Omitted => {
|
|
omitted_count = omitted_count.saturating_add(1);
|
|
truncated_description_chars =
|
|
truncated_description_chars.saturating_add(description_char_count);
|
|
if description_char_count > 0 {
|
|
truncated_description_count = truncated_description_count.saturating_add(1);
|
|
}
|
|
}
|
|
SkillLineAllocation::DescriptionChars(description_chars) => {
|
|
let truncated_chars = description_char_count.saturating_sub(*description_chars);
|
|
if truncated_chars > 0 {
|
|
truncated_description_chars =
|
|
truncated_description_chars.saturating_add(truncated_chars);
|
|
truncated_description_count = truncated_description_count.saturating_add(1);
|
|
}
|
|
lines.push(RenderedSkillLine {
|
|
line: line.render_with_description_chars(*description_chars),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
RenderedSkillLines {
|
|
lines,
|
|
omitted_count,
|
|
truncated_description_chars,
|
|
truncated_description_count,
|
|
}
|
|
}
|
|
|
|
fn allocate_description_chars(
|
|
budget: SkillMetadataBudget,
|
|
skill_lines: &[SkillLine<'_>],
|
|
limit: usize,
|
|
) -> Vec<usize> {
|
|
let budget_lines = skill_lines
|
|
.iter()
|
|
.map(|line| DescriptionBudgetLine::new(line, budget))
|
|
.collect::<Vec<_>>();
|
|
let mut char_allocations = vec![0usize; budget_lines.len()];
|
|
let mut current_extra_costs = vec![0usize; budget_lines.len()];
|
|
let mut remaining = limit;
|
|
|
|
// Distribute description space round-robin so no skill monopolizes the
|
|
// remaining budget.
|
|
loop {
|
|
let mut changed = false;
|
|
for (index, line) in budget_lines.iter().enumerate() {
|
|
if char_allocations[index] >= line.description_char_count {
|
|
continue;
|
|
}
|
|
|
|
let next_chars = char_allocations[index].saturating_add(1);
|
|
let next_cost = line.extra_costs[next_chars];
|
|
let delta = next_cost.saturating_sub(current_extra_costs[index]);
|
|
if delta <= remaining {
|
|
char_allocations[index] = next_chars;
|
|
current_extra_costs[index] = next_cost;
|
|
remaining = remaining.saturating_sub(delta);
|
|
changed = true;
|
|
}
|
|
}
|
|
|
|
if !changed {
|
|
break;
|
|
}
|
|
}
|
|
|
|
char_allocations
|
|
}
|
|
|
|
struct RenderedCatalog {
|
|
prompt_kind: SkillPromptKind,
|
|
skill_root_lines: Vec<String>,
|
|
skill_lines: Vec<String>,
|
|
report: SkillRenderReport,
|
|
}
|
|
|
|
pub(crate) struct AvailableSkillsRender {
|
|
prompt_kind: SkillPromptKind,
|
|
skill_root_lines: Vec<String>,
|
|
skill_lines: Vec<String>,
|
|
preserve_empty_fragment: bool,
|
|
pub(crate) report: SkillRenderReport,
|
|
}
|
|
|
|
#[derive(Default)]
|
|
pub(crate) struct RenderedSkillCatalogs {
|
|
pub(crate) executor: Option<AvailableSkillsRender>,
|
|
pub(crate) cloud: Option<AvailableSkillsRender>,
|
|
pub(crate) host: Option<AvailableSkillsRender>,
|
|
}
|
|
|
|
#[path = "render_dedup.rs"]
|
|
mod dedup;
|
|
pub(crate) use dedup::PreparedSkillCatalog;
|
|
|
|
impl AvailableSkillsRender {
|
|
/// Count the metadata charged by the allocator, including aliases and omission notices.
|
|
pub(crate) fn metadata_cost(
|
|
&self,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> usize {
|
|
let root_cost = if self.skill_root_lines.is_empty() {
|
|
0
|
|
} else {
|
|
aliased_metadata_overhead_cost(
|
|
budget,
|
|
self.prompt_kind,
|
|
&self.skill_root_lines,
|
|
include_skills_usage_instructions,
|
|
)
|
|
};
|
|
self.skill_lines.iter().fold(root_cost, |used, line| {
|
|
used.saturating_add(metadata_line_cost(budget, line))
|
|
})
|
|
}
|
|
|
|
pub(crate) fn into_fragment(
|
|
self,
|
|
include_skills_usage_instructions: bool,
|
|
) -> Option<AvailableSkillsInstructions> {
|
|
(self.preserve_empty_fragment || !self.skill_lines.is_empty()).then(|| {
|
|
AvailableSkillsInstructions::from_skill_lines(
|
|
self.prompt_kind,
|
|
self.skill_root_lines,
|
|
self.skill_lines,
|
|
include_skills_usage_instructions,
|
|
)
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tracing::instrument(
|
|
level = "trace",
|
|
skip_all,
|
|
fields(catalog_entry_count = catalog.entries.len())
|
|
)]
|
|
pub(crate) fn render_available_skills(
|
|
catalog: &SkillCatalog,
|
|
policy: SkillCatalogRenderPolicy,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> Option<AvailableSkillsRender> {
|
|
PreparedSkillCatalog::new(catalog, policy).render(budget, include_skills_usage_instructions)
|
|
}
|
|
|
|
pub(crate) fn render_prepared_skill_catalogs(
|
|
executor: &PreparedSkillCatalog<'_>,
|
|
cloud: &PreparedSkillCatalog<'_>,
|
|
host: &PreparedSkillCatalog<'_>,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> RenderedSkillCatalogs {
|
|
let nonempty_catalog_count = [
|
|
!executor.entries.is_empty(),
|
|
!cloud.entries.is_empty(),
|
|
!host.entries.is_empty(),
|
|
]
|
|
.into_iter()
|
|
.filter(|nonempty| *nonempty)
|
|
.count();
|
|
if nonempty_catalog_count <= 1 {
|
|
return RenderedSkillCatalogs {
|
|
executor: executor.render(budget, include_skills_usage_instructions),
|
|
cloud: cloud.render(budget, include_skills_usage_instructions),
|
|
host: host.render(budget, include_skills_usage_instructions),
|
|
};
|
|
}
|
|
|
|
let absolute = render_combined_lines(
|
|
executor.unaliased(),
|
|
cloud.unaliased(),
|
|
host.unaliased(),
|
|
budget,
|
|
);
|
|
|
|
let mut selected = absolute;
|
|
let host_only_aliases = build_aliased_combined_catalog(
|
|
executor.unaliased(),
|
|
cloud.unaliased(),
|
|
host.aliased(),
|
|
budget,
|
|
include_skills_usage_instructions,
|
|
);
|
|
let executor_only_aliases = build_aliased_combined_catalog(
|
|
executor.aliased(),
|
|
cloud.unaliased(),
|
|
host.unaliased(),
|
|
budget,
|
|
include_skills_usage_instructions,
|
|
);
|
|
let cloud_only_aliases = build_aliased_combined_catalog(
|
|
executor.unaliased(),
|
|
cloud.aliased(),
|
|
host.unaliased(),
|
|
budget,
|
|
include_skills_usage_instructions,
|
|
);
|
|
let all_source_aliases = build_aliased_combined_catalog(
|
|
executor.aliased(),
|
|
cloud.aliased(),
|
|
host.aliased(),
|
|
budget,
|
|
include_skills_usage_instructions,
|
|
);
|
|
|
|
for candidate in [
|
|
host_only_aliases,
|
|
executor_only_aliases,
|
|
cloud_only_aliases,
|
|
all_source_aliases,
|
|
]
|
|
.into_iter()
|
|
.flatten()
|
|
{
|
|
if combined_render_is_better(
|
|
&candidate,
|
|
&selected,
|
|
budget,
|
|
include_skills_usage_instructions,
|
|
) {
|
|
selected = candidate;
|
|
}
|
|
}
|
|
|
|
RenderedSkillCatalogs {
|
|
executor: Some(selected.executor),
|
|
cloud: Some(selected.cloud),
|
|
host: Some(selected.host),
|
|
}
|
|
}
|
|
|
|
struct CombinedAvailableSkillsRender {
|
|
executor: AvailableSkillsRender,
|
|
cloud: AvailableSkillsRender,
|
|
host: AvailableSkillsRender,
|
|
}
|
|
|
|
struct CatalogLines<'a> {
|
|
prompt_kind: SkillPromptKind,
|
|
skills: Vec<SkillLine<'a>>,
|
|
root_lines: Vec<String>,
|
|
}
|
|
|
|
impl<'a> CatalogLines<'a> {
|
|
fn unaliased(entries: &[&'a SkillCatalogEntry], policy: SkillCatalogRenderPolicy) -> Self {
|
|
Self {
|
|
prompt_kind: SkillPromptKind::Unaliased,
|
|
skills: entries
|
|
.iter()
|
|
.map(|entry| SkillLine::new(entry, policy))
|
|
.collect(),
|
|
root_lines: Vec::new(),
|
|
}
|
|
}
|
|
|
|
fn aliased(
|
|
entries: &[&'a SkillCatalogEntry],
|
|
policy: SkillCatalogRenderPolicy,
|
|
alias_plan: Option<&AliasPlan>,
|
|
) -> Self {
|
|
let Some(plan) = alias_plan.filter(|_| !entries.is_empty()) else {
|
|
return Self::unaliased(entries, policy);
|
|
};
|
|
|
|
Self {
|
|
prompt_kind: entries
|
|
.first()
|
|
.map(|entry| SkillPromptKind::for_aliased_source(&entry.authority.kind))
|
|
.unwrap_or(SkillPromptKind::Unaliased),
|
|
skills: entries
|
|
.iter()
|
|
.map(|entry| {
|
|
SkillLine::with_locator(
|
|
entry,
|
|
policy,
|
|
render_skill_locator_with_aliases(entry, plan),
|
|
)
|
|
})
|
|
.collect(),
|
|
root_lines: plan.root_lines(),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn render_combined_lines(
|
|
executor: CatalogLines<'_>,
|
|
cloud: CatalogLines<'_>,
|
|
host: CatalogLines<'_>,
|
|
budget: SkillMetadataBudget,
|
|
) -> CombinedAvailableSkillsRender {
|
|
let executor_end = executor.skills.len();
|
|
let cloud_end = executor_end.saturating_add(cloud.skills.len());
|
|
let mut lines = executor.skills;
|
|
lines.extend(cloud.skills);
|
|
lines.extend(host.skills);
|
|
let mut allocations = allocate_skill_lines(&lines, budget);
|
|
let omission_marker =
|
|
reserve_non_host_omission_marker(&lines, executor_end, cloud_end, budget, &mut allocations);
|
|
let (executor_omission_marker, cloud_omission_marker) = if executor_end == cloud_end {
|
|
(omission_marker, None)
|
|
} else {
|
|
(None, omission_marker)
|
|
};
|
|
|
|
CombinedAvailableSkillsRender {
|
|
executor: render_combined_group(
|
|
&lines[..executor_end],
|
|
&allocations[..executor_end],
|
|
executor.prompt_kind,
|
|
executor.root_lines,
|
|
executor_omission_marker,
|
|
),
|
|
cloud: render_combined_group(
|
|
&lines[executor_end..cloud_end],
|
|
&allocations[executor_end..cloud_end],
|
|
cloud.prompt_kind,
|
|
cloud.root_lines,
|
|
cloud_omission_marker,
|
|
),
|
|
host: render_combined_group(
|
|
&lines[cloud_end..],
|
|
&allocations[cloud_end..],
|
|
host.prompt_kind,
|
|
host.root_lines,
|
|
/*omission_marker*/ None,
|
|
),
|
|
}
|
|
}
|
|
|
|
fn render_combined_group(
|
|
skill_lines: &[SkillLine<'_>],
|
|
allocations: &[SkillLineAllocation],
|
|
prompt_kind: SkillPromptKind,
|
|
skill_root_lines: Vec<String>,
|
|
omission_marker: Option<String>,
|
|
) -> AvailableSkillsRender {
|
|
let RenderedSkillLines {
|
|
mut lines,
|
|
omitted_count,
|
|
truncated_description_chars,
|
|
truncated_description_count,
|
|
} = render_allocated_skill_lines(skill_lines, allocations);
|
|
if let Some(marker) = omission_marker {
|
|
lines.push(RenderedSkillLine { line: marker });
|
|
}
|
|
AvailableSkillsRender {
|
|
prompt_kind,
|
|
skill_root_lines,
|
|
skill_lines: lines.into_iter().map(|rendered| rendered.line).collect(),
|
|
preserve_empty_fragment: false,
|
|
report: SkillRenderReport {
|
|
total_count: skill_lines.len(),
|
|
included_count: skill_lines.len().saturating_sub(omitted_count),
|
|
omitted_count,
|
|
truncated_description_chars,
|
|
truncated_description_count,
|
|
},
|
|
}
|
|
}
|
|
|
|
fn build_aliased_combined_catalog(
|
|
executor: CatalogLines<'_>,
|
|
cloud: CatalogLines<'_>,
|
|
host: CatalogLines<'_>,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> Option<CombinedAvailableSkillsRender> {
|
|
if [&executor.root_lines, &cloud.root_lines, &host.root_lines]
|
|
.into_iter()
|
|
.all(Vec::is_empty)
|
|
{
|
|
return None;
|
|
}
|
|
|
|
let table_cost = [&executor, &cloud, &host]
|
|
.into_iter()
|
|
.filter(|catalog| !catalog.root_lines.is_empty())
|
|
.map(|catalog| {
|
|
aliased_metadata_overhead_cost(
|
|
budget,
|
|
catalog.prompt_kind,
|
|
&catalog.root_lines,
|
|
include_skills_usage_instructions,
|
|
)
|
|
})
|
|
.fold(0usize, usize::saturating_add);
|
|
if table_cost >= budget.limit() {
|
|
return None;
|
|
}
|
|
|
|
let adjusted_limit = budget.limit().saturating_sub(table_cost);
|
|
let adjusted_budget = match budget {
|
|
SkillMetadataBudget::Tokens(_) => SkillMetadataBudget::Tokens(adjusted_limit),
|
|
SkillMetadataBudget::Characters(_) => SkillMetadataBudget::Characters(adjusted_limit),
|
|
};
|
|
Some(render_combined_lines(
|
|
executor,
|
|
cloud,
|
|
host,
|
|
adjusted_budget,
|
|
))
|
|
}
|
|
|
|
fn combined_render_is_better(
|
|
candidate: &CombinedAvailableSkillsRender,
|
|
current: &CombinedAvailableSkillsRender,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> bool {
|
|
let priority = |rendered: &CombinedAvailableSkillsRender| {
|
|
(
|
|
rendered.executor.report.included_count,
|
|
rendered.cloud.report.included_count,
|
|
rendered.host.report.included_count,
|
|
)
|
|
};
|
|
if priority(candidate) != priority(current) {
|
|
return priority(candidate) > priority(current);
|
|
}
|
|
|
|
let truncated_chars = |rendered: &CombinedAvailableSkillsRender| {
|
|
[&rendered.executor, &rendered.cloud, &rendered.host]
|
|
.into_iter()
|
|
.fold(0usize, |total, catalog| {
|
|
total.saturating_add(catalog.report.truncated_description_chars)
|
|
})
|
|
};
|
|
if truncated_chars(candidate) != truncated_chars(current) {
|
|
return truncated_chars(candidate) < truncated_chars(current);
|
|
}
|
|
|
|
combined_available_skills_cost(budget, candidate, include_skills_usage_instructions)
|
|
< combined_available_skills_cost(budget, current, include_skills_usage_instructions)
|
|
}
|
|
|
|
fn combined_available_skills_cost(
|
|
budget: SkillMetadataBudget,
|
|
rendered: &CombinedAvailableSkillsRender,
|
|
include_skills_usage_instructions: bool,
|
|
) -> usize {
|
|
[&rendered.executor, &rendered.cloud, &rendered.host]
|
|
.into_iter()
|
|
.fold(0usize, |used, catalog| {
|
|
used.saturating_add(catalog.metadata_cost(budget, include_skills_usage_instructions))
|
|
})
|
|
}
|
|
|
|
fn reserve_non_host_omission_marker(
|
|
skill_lines: &[SkillLine<'_>],
|
|
executor_end: usize,
|
|
cloud_end: usize,
|
|
budget: SkillMetadataBudget,
|
|
allocations: &mut [SkillLineAllocation],
|
|
) -> Option<String> {
|
|
loop {
|
|
let omitted_count = allocations[..cloud_end]
|
|
.iter()
|
|
.filter(|allocation| matches!(allocation, SkillLineAllocation::Omitted))
|
|
.count();
|
|
if omitted_count == 0 {
|
|
return None;
|
|
}
|
|
|
|
let marker = omission_marker(omitted_count);
|
|
let used = allocated_skill_lines_cost(skill_lines, allocations, budget);
|
|
if used.saturating_add(metadata_line_cost(budget, &marker)) <= budget.limit() {
|
|
return Some(marker);
|
|
}
|
|
|
|
let index = (cloud_end..allocations.len())
|
|
.rev()
|
|
.chain((executor_end..cloud_end).rev())
|
|
.chain((0..executor_end).rev())
|
|
.find(|index| {
|
|
matches!(
|
|
allocations[*index],
|
|
SkillLineAllocation::DescriptionChars(_)
|
|
)
|
|
})?;
|
|
allocations[index] = SkillLineAllocation::Omitted;
|
|
}
|
|
}
|
|
|
|
fn allocated_skill_lines_cost(
|
|
skill_lines: &[SkillLine<'_>],
|
|
allocations: &[SkillLineAllocation],
|
|
budget: SkillMetadataBudget,
|
|
) -> usize {
|
|
skill_lines
|
|
.iter()
|
|
.zip(allocations)
|
|
.fold(0usize, |used, (line, allocation)| match allocation {
|
|
SkillLineAllocation::Omitted => used,
|
|
SkillLineAllocation::DescriptionChars(description_chars) => {
|
|
used.saturating_add(metadata_line_cost(
|
|
budget,
|
|
&line.render_with_description_chars(*description_chars),
|
|
))
|
|
}
|
|
})
|
|
}
|
|
|
|
fn render_catalog(
|
|
skill_lines: Vec<SkillLine<'_>>,
|
|
budget: SkillMetadataBudget,
|
|
skill_root_lines: Vec<String>,
|
|
prompt_kind: SkillPromptKind,
|
|
policy: SkillCatalogRenderPolicy,
|
|
) -> RenderedCatalog {
|
|
let total_count = skill_lines.len();
|
|
let RenderedSkillLines {
|
|
lines: mut rendered_lines,
|
|
omitted_count: mut omitted,
|
|
truncated_description_chars,
|
|
truncated_description_count,
|
|
} = render_skill_lines(skill_lines, budget);
|
|
let mut total_cost = rendered_lines.iter().fold(0usize, |used, rendered| {
|
|
used.saturating_add(metadata_line_cost(budget, &rendered.line))
|
|
});
|
|
|
|
if omitted > 0 && policy.includes_omission_notice() {
|
|
loop {
|
|
let marker = omission_marker(omitted);
|
|
if total_cost.saturating_add(metadata_line_cost(budget, &marker)) <= budget.limit() {
|
|
rendered_lines.push(RenderedSkillLine { line: marker });
|
|
break;
|
|
}
|
|
let Some(rendered) = rendered_lines.pop() else {
|
|
break;
|
|
};
|
|
total_cost = total_cost.saturating_sub(metadata_line_cost(budget, &rendered.line));
|
|
omitted = omitted.saturating_add(1);
|
|
}
|
|
}
|
|
|
|
RenderedCatalog {
|
|
prompt_kind,
|
|
skill_root_lines,
|
|
skill_lines: rendered_lines
|
|
.into_iter()
|
|
.map(|rendered| rendered.line)
|
|
.collect(),
|
|
report: SkillRenderReport {
|
|
total_count,
|
|
included_count: total_count.saturating_sub(omitted),
|
|
omitted_count: omitted,
|
|
truncated_description_chars,
|
|
truncated_description_count,
|
|
},
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn available_skills_fragment(
|
|
catalog: &SkillCatalog,
|
|
include_skills_usage_instructions: bool,
|
|
policy: SkillCatalogRenderPolicy,
|
|
budget: SkillMetadataBudget,
|
|
) -> Option<AvailableSkillsInstructions> {
|
|
render_available_skills(catalog, policy, budget, include_skills_usage_instructions)?
|
|
.into_fragment(include_skills_usage_instructions)
|
|
}
|
|
|
|
fn build_aliased_catalog(
|
|
catalog: CatalogLines<'_>,
|
|
policy: SkillCatalogRenderPolicy,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> Option<RenderedCatalog> {
|
|
if catalog.root_lines.is_empty() {
|
|
return None;
|
|
}
|
|
let table_cost = aliased_metadata_overhead_cost(
|
|
budget,
|
|
catalog.prompt_kind,
|
|
&catalog.root_lines,
|
|
include_skills_usage_instructions,
|
|
);
|
|
if table_cost >= budget.limit() {
|
|
return None;
|
|
}
|
|
|
|
let adjusted_limit = budget.limit().saturating_sub(table_cost);
|
|
let adjusted_budget = match budget {
|
|
SkillMetadataBudget::Tokens(_) => SkillMetadataBudget::Tokens(adjusted_limit),
|
|
SkillMetadataBudget::Characters(_) => SkillMetadataBudget::Characters(adjusted_limit),
|
|
};
|
|
Some(render_catalog(
|
|
catalog.skills,
|
|
adjusted_budget,
|
|
catalog.root_lines,
|
|
catalog.prompt_kind,
|
|
policy,
|
|
))
|
|
}
|
|
|
|
pub(crate) fn build_alias_plan(entries: &[&SkillCatalogEntry]) -> Option<AliasPlan> {
|
|
let source = &entries.first()?.authority.kind;
|
|
if entries.iter().any(|entry| &entry.authority.kind != source) {
|
|
return None;
|
|
}
|
|
let mut alias_ordered_entries = entries.to_vec();
|
|
alias_ordered_entries.sort_by_key(|entry| entry.alias_root_order().unwrap_or(usize::MAX));
|
|
let roots = match source {
|
|
SkillSourceKind::Host => shared_host_alias_roots(&alias_ordered_entries),
|
|
SkillSourceKind::Executor | SkillSourceKind::Cloud => alias_ordered_entries
|
|
.iter()
|
|
.filter_map(|entry| entry.alias_root())
|
|
.map(str::to_string)
|
|
.collect(),
|
|
SkillSourceKind::Custom(_) => return None,
|
|
};
|
|
let roots = roots.iter().map(String::as_str).collect::<Vec<_>>();
|
|
|
|
AliasPlan::build(source, &roots)
|
|
}
|
|
|
|
fn render_skill_locator_with_aliases(entry: &SkillCatalogEntry, plan: &AliasPlan) -> String {
|
|
let locator = match &entry.authority.kind {
|
|
SkillSourceKind::Executor | SkillSourceKind::Cloud => entry.id.0.as_str(),
|
|
SkillSourceKind::Host | SkillSourceKind::Custom(_) => entry.rendered_path(),
|
|
};
|
|
if entry.alias_root().is_none() {
|
|
return locator.to_string();
|
|
}
|
|
plan.shorten(locator).unwrap_or_else(|| locator.to_string())
|
|
}
|
|
|
|
fn aliased_metadata_overhead_cost(
|
|
budget: SkillMetadataBudget,
|
|
prompt_kind: SkillPromptKind,
|
|
skill_root_lines: &[String],
|
|
include_skills_usage_instructions: bool,
|
|
) -> usize {
|
|
let empty_skill_lines: &[String] = &[];
|
|
let absolute_body =
|
|
render_available_skills_body(SkillPromptKind::Unaliased, &[], empty_skill_lines);
|
|
let aliased_body =
|
|
render_available_skills_body(prompt_kind, skill_root_lines, empty_skill_lines);
|
|
let alias_instruction_cost = if include_skills_usage_instructions {
|
|
prompt_kind
|
|
.alias_instructions()
|
|
.map_or(0, |instructions| metadata_line_cost(budget, instructions))
|
|
} else {
|
|
0
|
|
};
|
|
budget
|
|
.cost(&aliased_body)
|
|
.saturating_add(alias_instruction_cost)
|
|
.saturating_sub(budget.cost(&absolute_body))
|
|
}
|
|
|
|
fn aliased_render_is_better(
|
|
aliased: &RenderedCatalog,
|
|
absolute: &RenderedCatalog,
|
|
budget: SkillMetadataBudget,
|
|
include_skills_usage_instructions: bool,
|
|
) -> bool {
|
|
if aliased.report.included_count != absolute.report.included_count {
|
|
return aliased.report.included_count > absolute.report.included_count;
|
|
}
|
|
if aliased.report.truncated_description_chars != absolute.report.truncated_description_chars {
|
|
return aliased.report.truncated_description_chars
|
|
< absolute.report.truncated_description_chars;
|
|
}
|
|
rendered_catalog_cost(budget, aliased, include_skills_usage_instructions)
|
|
< rendered_catalog_cost(budget, absolute, include_skills_usage_instructions)
|
|
}
|
|
|
|
fn rendered_catalog_cost(
|
|
budget: SkillMetadataBudget,
|
|
rendered: &RenderedCatalog,
|
|
include_skills_usage_instructions: bool,
|
|
) -> usize {
|
|
let metadata_cost = if rendered.skill_root_lines.is_empty() {
|
|
0
|
|
} else {
|
|
aliased_metadata_overhead_cost(
|
|
budget,
|
|
rendered.prompt_kind,
|
|
&rendered.skill_root_lines,
|
|
include_skills_usage_instructions,
|
|
)
|
|
};
|
|
rendered
|
|
.skill_lines
|
|
.iter()
|
|
.fold(metadata_cost, |used, line| {
|
|
used.saturating_add(metadata_line_cost(budget, line))
|
|
})
|
|
}
|
|
|
|
fn omission_marker(omitted: usize) -> String {
|
|
let skill_word = if omitted == 1 { "skill" } else { "skills" };
|
|
format!("- {omitted} additional {skill_word} omitted from this bounded skills list.")
|
|
}
|
|
|
|
pub(crate) fn truncate_catalog_skill_description(description: &str) -> Cow<'_, str> {
|
|
if description
|
|
.char_indices()
|
|
.nth(MAX_CATALOG_SKILL_DESCRIPTION_CHARS)
|
|
.is_none()
|
|
{
|
|
return Cow::Borrowed(description);
|
|
}
|
|
|
|
let prefix_chars = MAX_CATALOG_SKILL_DESCRIPTION_CHARS
|
|
.saturating_sub(TRUNCATED_SKILL_DESCRIPTION_SUFFIX.chars().count());
|
|
let prefix_end = description
|
|
.char_indices()
|
|
.nth(prefix_chars)
|
|
.map_or(description.len(), |(index, _)| index);
|
|
let mut truncated = description[..prefix_end].to_string();
|
|
truncated.push_str(TRUNCATED_SKILL_DESCRIPTION_SUFFIX);
|
|
Cow::Owned(truncated)
|
|
}
|
|
|
|
pub(crate) fn truncate_main_prompt_contents(contents: &str) -> (String, bool) {
|
|
truncate_utf8_to_bytes(contents, MAX_SKILL_PROMPT_BYTES)
|
|
}
|
|
|
|
pub(crate) fn truncate_utf8_to_bytes(contents: &str, max_bytes: usize) -> (String, bool) {
|
|
let truncated = take_bytes_at_char_boundary(contents, max_bytes);
|
|
(truncated.to_string(), truncated.len() < contents.len())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "render_tests.rs"]
|
|
mod tests;
|