Scale skill metadata budgets with model context windows (#34626)

## Why

A fixed character limit does not account for the different context-window sizes supported by models.

## What changed

- Budget extension-rendered skill metadata at 2% of the resolved model context window, capped at 4,000 tokens.
- Keep the existing 8,000-character fallback when model context metadata is unavailable.
- Include the omission marker in the budget and still emit it when no skill entry fits.
- Apply the same resolved budget to executor and host skill catalogs assembled for a turn.

## Testing

- Cover proportional and capped budgets, multibyte fallback accounting, and omission-marker behavior.
- Verify through the production turn path that larger model context windows include more catalog entries without exceeding the computed budget.

GitOrigin-RevId: 4667293f1594de4dd605b32b9fa4255d26772c0f
This commit is contained in:
felixxia-oai
2026-07-21 22:03:50 +00:00
committed by copyberry
parent bbad09a83b
commit 37eef7bacc
10 changed files with 454 additions and 31 deletions

2
codex-rs/Cargo.lock generated
View File

@@ -2672,6 +2672,7 @@ dependencies = [
"codex-shell-command",
"codex-shell-escalation",
"codex-skills",
"codex-skills-extension",
"codex-state",
"codex-terminal-detection",
"codex-test-binary-support",
@@ -3976,6 +3977,7 @@ dependencies = [
"codex-exec-server",
"codex-extension-api",
"codex-mcp",
"codex-models-manager",
"codex-otel",
"codex-protocol",
"codex-skills",

View File

@@ -141,6 +141,7 @@ assert_matches = { workspace = true }
codex-image-generation-extension = { workspace = true }
codex-home = { workspace = true }
codex-otel = { workspace = true }
codex-skills-extension = { workspace = true }
codex-test-binary-support = { workspace = true }
codex-utils-cargo-bin = { workspace = true }
codex-web-search-extension = { workspace = true }

View File

@@ -130,6 +130,7 @@ mod shell_serialization;
mod shell_snapshot;
mod skill_approval;
mod skills;
mod skills_extension;
mod spawn_agent_description;
mod sqlite_state;
mod stream_error_allows_next_turn;

View File

@@ -0,0 +1,151 @@
use std::sync::Arc;
use anyhow::Result;
use codex_core::config::Config;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_skills_extension::SkillProvider;
use codex_skills_extension::SkillProviderSource;
use codex_skills_extension::SkillProviders;
use codex_skills_extension::SkillsExtensionConfig;
use codex_skills_extension::catalog::SkillAuthority;
use codex_skills_extension::catalog::SkillCatalog;
use codex_skills_extension::catalog::SkillCatalogEntry;
use codex_skills_extension::catalog::SkillPackageId;
use codex_skills_extension::catalog::SkillProviderError;
use codex_skills_extension::catalog::SkillReadResult;
use codex_skills_extension::catalog::SkillResourceId;
use codex_skills_extension::catalog::SkillSearchResult;
use codex_skills_extension::catalog::SkillSourceKind;
use codex_skills_extension::install_with_providers;
use codex_skills_extension::provider::SkillListQuery;
use codex_skills_extension::provider::SkillProviderFuture;
use codex_skills_extension::provider::SkillReadRequest;
use codex_skills_extension::provider::SkillSearchRequest;
use codex_utils_string::approx_token_count;
use core_test_support::responses;
use core_test_support::responses::ev_completed;
use core_test_support::responses::ev_response_created;
use core_test_support::responses::mount_sse_once;
use core_test_support::responses::sse;
use core_test_support::test_codex::test_codex;
struct StaticSkillProvider {
catalog: SkillCatalog,
}
impl SkillProvider for StaticSkillProvider {
fn list(&self, query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> {
// Keep thread context empty so the catalog is exercised through the
// production turn-input path, where the host snapshot is available.
let catalog = if query.host_snapshot.is_some() {
self.catalog.clone()
} else {
SkillCatalog::default()
};
Box::pin(async move { Ok(catalog) })
}
fn read(&self, _request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> {
Box::pin(async {
Err(SkillProviderError::new(
"production-flow catalog test does not read skills",
))
})
}
fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> {
Box::pin(async { Ok(SkillSearchResult::default()) })
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn production_turn_scales_extension_catalog_from_resolved_model_window() -> Result<()> {
let mut included_counts = Vec::new();
for (context_window, max_context_window, expected_budget) in
[(Some(10_000), None, 200), (None, Some(400_000), 4_000)]
{
let server = responses::start_mock_server().await;
let response = mount_sse_once(
&server,
sse(vec![ev_response_created("resp-1"), ev_completed("resp-1")]),
)
.await;
let source_kind = SkillSourceKind::Custom("test".to_string());
let catalog = SkillCatalog {
entries: (0..400)
.map(|index| {
let name = format!("skill-{index:03}");
SkillCatalogEntry::new(
SkillPackageId(format!("test/{name}")),
SkillAuthority::new(source_kind.clone(), "test"),
name.clone(),
"A description long enough to keep the catalog under sustained budget pressure.",
SkillResourceId::new(format!("{name}/SKILL.md")),
)
.with_display_path(format!("skill://test/{name}/SKILL.md"))
})
.collect(),
warnings: Vec::new(),
};
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
install_with_providers(
&mut extensions,
SkillProviders::new().with_provider(SkillProviderSource::new(
source_kind,
"test",
Arc::new(StaticSkillProvider { catalog }),
)),
|config: &Config| SkillsExtensionConfig {
include_instructions: config.include_skill_instructions,
bundled_skills_enabled: false,
orchestrator_skills_enabled: false,
shadow_selection_enabled: false,
},
);
let mut builder = test_codex()
.with_extensions(Arc::new(extensions.build()))
.with_model_info_override("gpt-5.5", move |model_info| {
model_info.context_window = context_window;
model_info.max_context_window = max_context_window;
})
.with_config(|config| {
config.include_skill_instructions = true;
});
let test = builder.build_with_auto_env(&server).await?;
test.submit_turn("Inspect the available skills.").await?;
let request = response.single_request();
let developer_texts = request.message_input_texts("developer");
let catalog_text = developer_texts
.iter()
.find(|text| text.contains("skill://test/"))
.unwrap_or_else(|| {
panic!(
"production request should include the extension skill catalog, got {developer_texts:?}"
)
});
let metadata_lines = catalog_text
.lines()
.skip_while(|line| *line != "### Available skills")
.skip(1)
.take_while(|line| !line.starts_with("### "))
.filter(|line| line.starts_with("- "))
.collect::<Vec<_>>();
let metadata_cost = metadata_lines.iter().fold(0usize, |cost, line| {
cost.saturating_add(approx_token_count(&format!("{line}\n")))
});
let included_count = metadata_lines
.iter()
.filter(|line| line.starts_with("- skill-"))
.count();
assert!(catalog_text.contains("additional skills omitted"));
assert!(metadata_cost <= expected_budget);
included_counts.push(included_count);
}
assert!(included_counts[0] > 0);
assert!(included_counts[0] < included_counts[1]);
Ok(())
}

View File

@@ -31,6 +31,7 @@ tracing = { workspace = true }
url = { workspace = true }
[dev-dependencies]
codex-models-manager = { workspace = true }
codex-utils-absolute-path = { workspace = true }
pretty_assertions = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }

View File

@@ -1,7 +1,6 @@
use std::sync::Arc;
use codex_core_skills::HostSkillsSnapshot;
use codex_core_skills::default_skill_metadata_budget;
use codex_core_skills::injection::HostSkillsCatalogInWorldState;
use codex_core_skills::injection::InjectedHostSkillPrompts;
use codex_exec_server::LOCAL_ENVIRONMENT_ID;
@@ -45,6 +44,7 @@ use crate::render::MAX_SKILL_NAME_BYTES;
use crate::render::MAX_SKILL_PATH_BYTES;
use crate::render::SkillCatalogRenderPolicy;
use crate::render::available_skills_fragment;
use crate::render::capped_skill_metadata_budget;
use crate::render::truncate_main_prompt_contents;
use crate::render::truncate_utf8_to_bytes;
use crate::selection::collect_explicit_skill_mentions;
@@ -149,6 +149,7 @@ where
&catalog,
include_usage,
SkillCatalogRenderPolicy::ExtensionCompatible,
capped_skill_metadata_budget(/*context_window*/ None),
)
.map(|fragment| PromptFragment::developer_capability(fragment.render()))
.into_iter()
@@ -187,10 +188,15 @@ where
let include_usage = model_info
.as_deref()
.is_some_and(|model_info| model_info.include_skills_usage_instructions);
let context_window = model_info
.as_deref()
.and_then(ModelInfo::resolved_context_window);
let metadata_budget = capped_skill_metadata_budget(context_window);
let mut sections = vec![executor_skills_world_state_section(
&catalog,
config.include_instructions,
include_usage,
metadata_budget,
)];
if let Some(host_snapshot) = input.turn_store.get::<HostSkillsSnapshot>()
&& self.providers.has_host_provider()
@@ -200,11 +206,7 @@ where
&host_snapshot,
config.include_instructions,
include_usage,
default_skill_metadata_budget(
model_info
.as_deref()
.and_then(|model_info| model_info.context_window),
),
metadata_budget,
));
}
sections
@@ -331,13 +333,19 @@ where
entry.authority.kind != SkillSourceKind::Executor
&& entry.authority.kind != SkillSourceKind::Orchestrator
});
let include_usage = thread_store
.get::<ModelInfo>()
let model_info = thread_store.get::<ModelInfo>();
let include_usage = model_info
.as_deref()
.is_some_and(|model_info| model_info.include_skills_usage_instructions);
let context_window = model_info
.as_deref()
.and_then(ModelInfo::resolved_context_window);
let metadata_budget = capped_skill_metadata_budget(context_window);
if let Some(fragment) = available_skills_fragment(
&turn_catalog,
include_usage,
SkillCatalogRenderPolicy::ExtensionCompatible,
metadata_budget,
) {
fragments.push(Box::new(fragment));
}

View File

@@ -1,5 +1,6 @@
use std::borrow::Cow;
use codex_utils_string::approx_token_count;
use codex_utils_string::take_bytes_at_char_boundary;
use crate::catalog::SkillCatalog;
@@ -7,7 +8,9 @@ use crate::catalog::SkillCatalogEntry;
use crate::catalog::SkillSourceKind;
use crate::fragments::AvailableSkillsInstructions;
const MAX_AVAILABLE_SKILLS_BYTES: usize = 8_000;
const DEFAULT_SKILL_METADATA_CHAR_BUDGET: usize = 8_000;
const MAX_SKILL_METADATA_TOKEN_BUDGET: usize = 4_000;
const SKILL_METADATA_CONTEXT_WINDOW_PERCENT: usize = 2;
const MAX_MAIN_PROMPT_BYTES: usize = 8_000;
const MAX_CATALOG_SKILL_DESCRIPTION_CHARS: usize = 1_024;
const TRUNCATED_SKILL_DESCRIPTION_SUFFIX: &str = "...";
@@ -39,6 +42,37 @@ impl SkillCatalogRenderPolicy {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SkillMetadataBudget {
Tokens(usize),
Characters(usize),
}
pub(crate) fn capped_skill_metadata_budget(context_window: Option<i64>) -> SkillMetadataBudget {
context_window
.and_then(|window| usize::try_from(window).ok())
.filter(|window| *window > 0)
.map(|window| {
SkillMetadataBudget::Tokens(
window
.saturating_mul(SKILL_METADATA_CONTEXT_WINDOW_PERCENT)
.saturating_div(100)
.clamp(1, MAX_SKILL_METADATA_TOKEN_BUDGET),
)
})
.unwrap_or(SkillMetadataBudget::Characters(
DEFAULT_SKILL_METADATA_CHAR_BUDGET,
))
}
fn metadata_line_cost(budget: SkillMetadataBudget, line: &str) -> usize {
let line = format!("{line}\n");
match budget {
SkillMetadataBudget::Tokens(_) => approx_token_count(&line),
SkillMetadataBudget::Characters(_) => line.chars().count(),
}
}
#[tracing::instrument(
level = "trace",
skip_all,
@@ -48,8 +82,12 @@ pub(crate) fn available_skills_fragment(
catalog: &SkillCatalog,
include_skills_usage_instructions: bool,
policy: SkillCatalogRenderPolicy,
budget: SkillMetadataBudget,
) -> Option<AvailableSkillsInstructions> {
let mut total_bytes = 0usize;
let budget_limit = match budget {
SkillMetadataBudget::Tokens(limit) | SkillMetadataBudget::Characters(limit) => limit,
};
let mut total_cost = 0usize;
let mut omitted = 0usize;
let mut skill_lines = Vec::new();
@@ -61,29 +99,39 @@ pub(crate) fn available_skills_fragment(
let description = policy.description(entry);
let description = truncate_catalog_skill_description(description);
let line = render_skill_line(entry, description.as_ref());
let next_bytes = total_bytes.saturating_add(line.len());
if next_bytes > MAX_AVAILABLE_SKILLS_BYTES {
let next_cost = total_cost.saturating_add(metadata_line_cost(budget, &line));
if next_cost > budget_limit {
omitted = omitted.saturating_add(1);
continue;
}
total_bytes = next_bytes;
total_cost = next_cost;
skill_lines.push(line);
}
if skill_lines.is_empty() {
return None;
}
if omitted > 0 {
let skill_word = if omitted == 1 { "skill" } else { "skills" };
skill_lines.push(format!(
"- {omitted} additional {skill_word} omitted from this bounded skills list."
));
loop {
let marker = omission_marker(omitted);
if total_cost.saturating_add(metadata_line_cost(budget, &marker)) <= budget_limit {
skill_lines.push(marker);
break;
}
let line = skill_lines.pop()?;
total_cost = total_cost.saturating_sub(metadata_line_cost(budget, &line));
omitted = omitted.saturating_add(1);
}
}
Some(AvailableSkillsInstructions::from_skill_lines(
skill_lines,
include_skills_usage_instructions,
))
(!skill_lines.is_empty()).then(|| {
AvailableSkillsInstructions::from_skill_lines(
skill_lines,
include_skills_usage_instructions,
)
})
}
fn omission_marker(omitted: usize) -> String {
let skill_word = if omitted == 1 { "skill" } else { "skills" };
format!("- {omitted} additional {skill_word} omitted from this bounded skills list.")
}
pub(crate) fn truncate_catalog_skill_description(description: &str) -> Cow<'_, str> {

View File

@@ -35,12 +35,14 @@ fn description_selection_follows_render_policy() {
&catalog,
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::CoreCompatible,
SkillMetadataBudget::Characters(8_000),
)
.expect("catalog should render");
let extension = available_skills_fragment(
&catalog,
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::ExtensionCompatible,
SkillMetadataBudget::Characters(8_000),
)
.expect("catalog should render");
@@ -65,3 +67,110 @@ fn description_selection_follows_render_policy() {
)
);
}
#[test]
fn catalog_budget_uses_capped_context_percentage_or_character_fallback() {
assert_eq!(
capped_skill_metadata_budget(Some(100_000)),
SkillMetadataBudget::Tokens(2_000)
);
assert_eq!(
capped_skill_metadata_budget(Some(400_000)),
SkillMetadataBudget::Tokens(4_000)
);
assert_eq!(
capped_skill_metadata_budget(/*context_window*/ None),
SkillMetadataBudget::Characters(8_000)
);
}
#[test]
fn omission_marker_is_charged_to_catalog_budget() {
let catalog = SkillCatalog {
entries: (0..20)
.map(|index| {
entry(
&format!("skill-{index:02}"),
"A description long enough to put the catalog under budget pressure.",
/*short_description*/ None,
)
})
.collect(),
warnings: Vec::new(),
};
let fragment = available_skills_fragment(
&catalog,
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::ExtensionCompatible,
SkillMetadataBudget::Tokens(100),
)
.expect("catalog should render");
let rendered_metadata_cost = fragment
.body()
.lines()
.filter(|line| line.starts_with("- "))
.map(|line| approx_token_count(&format!("{line}\n")))
.sum::<usize>();
assert!(fragment.body().contains("additional skills omitted"));
assert!(rendered_metadata_cost <= 100);
}
#[test]
fn character_fallback_counts_multibyte_metadata_by_characters() {
let description = "💡".repeat(MAX_CATALOG_SKILL_DESCRIPTION_CHARS);
let catalog = SkillCatalog {
entries: vec![
entry(
"multibyte-one",
&description,
/*short_description*/ None,
),
entry(
"multibyte-two",
&description,
/*short_description*/ None,
),
],
warnings: Vec::new(),
};
let fragment = available_skills_fragment(
&catalog,
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::ExtensionCompatible,
SkillMetadataBudget::Characters(8_000),
)
.expect("catalog should render");
assert!(fragment.body().contains("multibyte-one"));
assert!(fragment.body().contains("multibyte-two"));
assert!(!fragment.body().contains("additional skills omitted"));
}
#[test]
fn catalog_emits_omission_marker_when_every_skill_exceeds_budget() {
let catalog = SkillCatalog {
entries: vec![entry(
"oversized",
&"x".repeat(MAX_CATALOG_SKILL_DESCRIPTION_CHARS),
/*short_description*/ None,
)],
warnings: Vec::new(),
};
let fragment = available_skills_fragment(
&catalog,
/*include_skills_usage_instructions*/ false,
SkillCatalogRenderPolicy::ExtensionCompatible,
SkillMetadataBudget::Tokens(100),
)
.expect("omission marker should fit");
assert!(!fragment.body().contains("- oversized:"));
assert!(
fragment
.body()
.contains("- 1 additional skill omitted from this bounded skills list.")
);
}

View File

@@ -1,5 +1,4 @@
use codex_core_skills::HostSkillsSnapshot;
use codex_core_skills::SkillMetadataBudget;
use codex_core_skills::build_available_skills;
use codex_core_skills::render::SkillRenderSideEffects;
use codex_extension_api::ContextualUserFragment;
@@ -13,12 +12,11 @@ use serde_json::json;
use crate::catalog::SkillCatalog;
use crate::fragments::AvailableSkillsInstructions;
use crate::render::SkillCatalogRenderPolicy;
use crate::render::SkillMetadataBudget;
use crate::render::available_skills_fragment;
pub(crate) const SKILLS_WORLD_STATE_ID: &str = "skills";
pub(crate) const HOST_SKILLS_WORLD_STATE_ID: &str = "host_skills";
const MAX_HOST_SKILLS_METADATA_CHARS: usize = 8_000;
const MAX_HOST_SKILLS_METADATA_TOKENS: usize = 4_000;
const NO_EXECUTOR_SKILLS_BODY: &str =
"\n## Skills update\nNo selected-environment skills are currently available.\n";
const HIDDEN_EXECUTOR_SKILLS_BODY: &str = "\n## Skills update\nSelected-environment skills are not listed automatically. Explicit skill mentions can still be resolved when available.\n";
@@ -30,12 +28,14 @@ pub(crate) fn executor_skills_world_state_section(
catalog: &SkillCatalog,
include_instructions: bool,
include_skills_usage_instructions: bool,
metadata_budget: SkillMetadataBudget,
) -> WorldStateSectionContribution {
let body = if include_instructions {
available_skills_fragment(
catalog,
include_skills_usage_instructions,
SkillCatalogRenderPolicy::ExtensionCompatible,
metadata_budget,
)
.map(|fragment| fragment.body())
} else {
@@ -95,11 +95,9 @@ pub(crate) fn host_skills_world_state_section(
) -> WorldStateSectionContribution {
let outcome = host_snapshot.outcome();
let metadata_budget = match metadata_budget {
SkillMetadataBudget::Tokens(limit) => {
SkillMetadataBudget::Tokens(limit.min(MAX_HOST_SKILLS_METADATA_TOKENS))
}
SkillMetadataBudget::Tokens(limit) => codex_core_skills::SkillMetadataBudget::Tokens(limit),
SkillMetadataBudget::Characters(limit) => {
SkillMetadataBudget::Characters(limit.min(MAX_HOST_SKILLS_METADATA_CHARS))
codex_core_skills::SkillMetadataBudget::Characters(limit)
}
};
let available = if include_instructions {

View File

@@ -19,6 +19,7 @@ use codex_extension_api::ToolCall;
use codex_extension_api::ToolPayload;
use codex_extension_api::TurnInputContext;
use codex_extension_api::WorldStateContributionInput;
use codex_models_manager::model_info::model_info_from_slug;
use codex_protocol::capabilities::CapabilityRootLocation;
use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::protocol::Event;
@@ -652,6 +653,109 @@ async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> Te
Ok(())
}
#[tokio::test]
async fn model_context_window_scales_executor_catalog_but_not_thread_catalog() -> TestResult {
let orchestrator_entries = (0..40)
.map(|index| {
test_entry(
SkillSourceKind::Orchestrator,
"orchestrator",
&format!("orchestrator/skill-{index:02}"),
&format!("skill-{index:02}/SKILL.md"),
)
})
.collect();
let executor_entries = (0..200)
.map(|index| {
test_entry(
SkillSourceKind::Executor,
"env-1",
&format!("executor/skill-{index:02}"),
&format!("skill-{index:02}/SKILL.md"),
)
})
.collect();
let providers = SkillProviders::new()
.with_orchestrator_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
entries: orchestrator_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}))
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
entries: executor_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.bundled_skills_enabled = false;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let mut model_info = model_info_from_slug("test-model");
model_info.context_window = Some(10_000);
thread_store.insert(model_info);
let thread_fragments = registry.context_contributors()[0]
.contribute_thread_context(&session_store, &thread_store, &ExtensionData::new("step"))
.await;
assert_eq!(1, thread_fragments.len());
assert!(thread_fragments[0].text().contains("skill-39"));
assert!(
!thread_fragments[0]
.text()
.contains("additional skills omitted")
);
let selected_roots = vec![SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills").expect("skill root URI"),
},
}];
let turn_store = ExtensionData::new("turn-1");
let sections = registry.context_contributors()[0]
.contribute_world_state(WorldStateContributionInput {
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
ready_selected_capability_roots: &selected_roots,
executor_capability_discovery: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
step_store: &ExtensionData::new("step"),
})
.await;
let fragment = sections[0]
.render_diff(PreviousWorldStateSection::Absent)
.ok_or("bounded executor catalog should render")?;
assert!(fragment.body().contains("additional skills omitted"));
assert!(!fragment.body().contains("skill-39"));
Ok(())
}
#[tokio::test]
async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult {
let read_requests = Arc::new(Mutex::new(Vec::new()));