mirror of
https://github.com/openai/codex.git
synced 2026-10-19 17:28:46 +00:00
Deduplicate cloud and executor skill listings before budgeting (#49127)
## Why Skills available from both cloud and executor providers consume duplicate catalog space, leaving less room for descriptions of unique skills. ## What changed - Prefer model-visible cloud skills over executor listings with the same `plugin:skill` name before allocating the metadata budget. - Preserve aliases from the full visible executor inventory so filtering duplicates does not renumber remaining packages. Continue choosing full locators when they fit the budget better. - Keep executor packages available through `skills.list` and `skills.read`, and retain executor listings when cloud skills are disabled. ## Testing Add regression tests and a model-request snapshot covering description retention, stable aliases across turns, executor availability changes, package reads, and full-locator selection after deduplication. GitOrigin-RevId: 477e4cf16b8968dbe61438dd5af1eca9cbfa78c8
This commit is contained in:
Anthony Tafoya
committed by
copyberry
copyberry
parent
8bd5a136ff
commit
13f580ef09
@@ -130,6 +130,9 @@ mod mxc;
|
||||
#[path = "scenarios_tools_namespace_budget.rs"]
|
||||
mod tools_namespace_budget;
|
||||
|
||||
#[path = "scenarios_skill_catalog_dedup.rs"]
|
||||
mod skill_catalog_dedup;
|
||||
|
||||
fn skills_extensions() -> Arc<ExtensionRegistry<Config>> {
|
||||
let mut extensions = ExtensionRegistryBuilder::<Config>::new();
|
||||
install(&mut extensions, |config: &Config| SkillsExtensionConfig {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Snapshots cloud-preferred skill deduplication and stable executor aliases in model requests.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use codex_core::config::Config;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_features::Feature;
|
||||
use codex_skills_extension::SkillProviders;
|
||||
use codex_skills_extension::SkillsExtensionConfig;
|
||||
use codex_skills_extension::catalog::SkillAuthority;
|
||||
use codex_skills_extension::catalog::SkillCatalog;
|
||||
use codex_skills_extension::catalog::SkillCatalogEntry;
|
||||
use codex_skills_extension::catalog::SkillPackageId;
|
||||
use codex_skills_extension::catalog::SkillResourceId;
|
||||
use codex_skills_extension::catalog::SkillSourceKind;
|
||||
use codex_skills_extension::install_with_providers;
|
||||
use core_test_support::context_snapshot;
|
||||
use core_test_support::context_snapshot::ContextSnapshotOptions;
|
||||
use core_test_support::responses;
|
||||
use core_test_support::skip_if_no_network;
|
||||
use core_test_support::test_codex::test_codex;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use super::super::skills_extension::CatalogSkillProvider;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cloud_preference_preserves_executor_aliases_and_description_budget() -> Result<()> {
|
||||
skip_if_no_network!(Ok(()));
|
||||
|
||||
let server = responses::start_mock_server().await;
|
||||
let roots = [
|
||||
format!("skill://executor-a/{}", "a".repeat(180)),
|
||||
format!("skill://executor-b/{}", "b".repeat(180)),
|
||||
];
|
||||
let description = format!("{} UNIQUE_DESCRIPTION_END", "Skill guidance. ".repeat(60));
|
||||
let mut executor_entries = Vec::new();
|
||||
for (plugin, root) in ["demo", "other"].into_iter().zip(&roots) {
|
||||
for index in 0..3 {
|
||||
let package = format!("{root}/s{index}");
|
||||
executor_entries.push(
|
||||
SkillCatalogEntry::new(
|
||||
SkillPackageId(package.clone()),
|
||||
SkillAuthority::new(SkillSourceKind::Executor, "executor"),
|
||||
format!("{plugin}:s{index}"),
|
||||
&description,
|
||||
SkillResourceId::new(format!("{package}/SKILL.md")),
|
||||
)
|
||||
.with_alias_root(root),
|
||||
);
|
||||
}
|
||||
}
|
||||
let cloud_entries = (0..3)
|
||||
.map(|index| {
|
||||
SkillCatalogEntry::new(
|
||||
SkillPackageId(format!("skill://cloud/s{index}")),
|
||||
SkillAuthority::new(SkillSourceKind::Cloud, "cloud"),
|
||||
format!("demo:s{index}"),
|
||||
"Cloud skill instructions.",
|
||||
SkillResourceId::new(format!("skill://cloud/s{index}/SKILL.md")),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let provider = |entries| {
|
||||
Arc::new(CatalogSkillProvider {
|
||||
catalog: SkillCatalog {
|
||||
entries,
|
||||
warnings: Vec::new(),
|
||||
},
|
||||
})
|
||||
};
|
||||
let mut extensions = ExtensionRegistryBuilder::new();
|
||||
install_with_providers(
|
||||
&mut extensions,
|
||||
SkillProviders::new()
|
||||
.with_executor_provider(provider(executor_entries))
|
||||
.with_cloud_provider(provider(cloud_entries)),
|
||||
|config: &Config| SkillsExtensionConfig {
|
||||
include_instructions: config.include_skill_instructions,
|
||||
max_context_tokens: config.skill_max_context_tokens,
|
||||
bundled_skills_enabled: false,
|
||||
cloud_skill_enabled: true,
|
||||
shadow_selection_enabled: false,
|
||||
},
|
||||
);
|
||||
let mock = responses::mount_sse_sequence(
|
||||
&server,
|
||||
(0..2)
|
||||
.map(|index| {
|
||||
responses::sse(vec![
|
||||
responses::ev_assistant_message(
|
||||
&format!("message-{index}"),
|
||||
"Skills are available.",
|
||||
),
|
||||
responses::ev_completed(&format!("response-{index}")),
|
||||
])
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.await;
|
||||
let test = test_codex()
|
||||
.with_exec_server_url("none")
|
||||
.with_extensions(Arc::new(extensions.build()))
|
||||
.with_model_info_override("gpt-6-astra", |model| {
|
||||
model.context_window = Some(70_000);
|
||||
model.max_context_window = None;
|
||||
model.include_skills_usage_instructions = false;
|
||||
})
|
||||
.with_config(|config| {
|
||||
super::configure_scenario_catalog(config);
|
||||
config.cloud_skill_enabled = true;
|
||||
config
|
||||
.features
|
||||
.enable(Feature::ExecutorCapabilityDiscovery)
|
||||
.expect("enable executor capability discovery");
|
||||
})
|
||||
.build_with_auto_env(&server)
|
||||
.await?;
|
||||
test.submit_turn("Which skills are available?").await?;
|
||||
test.submit_turn("Are those same skills still available?")
|
||||
.await?;
|
||||
|
||||
let requests = mock.requests();
|
||||
assert_eq!(requests.len(), 2);
|
||||
for (request_index, request) in requests.iter().enumerate() {
|
||||
let body = request.message_input_texts("developer").join("\n");
|
||||
for index in 0..3 {
|
||||
assert_eq!(
|
||||
body.matches(&format!("- demo:s{index}:")).count(),
|
||||
1,
|
||||
"request {request_index}"
|
||||
);
|
||||
assert!(body.contains(&format!("(cloud package: skill://cloud/s{index})")));
|
||||
assert!(body.contains(&format!("(executor package: e1/s{index})")));
|
||||
assert!(!body.contains(&format!("(executor package: e0/s{index})")));
|
||||
}
|
||||
for (index, root) in roots.iter().enumerate() {
|
||||
assert!(body.contains(&format!("- `e{index}` = `{root}`")));
|
||||
}
|
||||
assert_eq!(body.matches("UNIQUE_DESCRIPTION_END").count(), 3);
|
||||
}
|
||||
insta::assert_snapshot!(
|
||||
"dedup_before_budgeting",
|
||||
context_snapshot::format_request_history_snapshot(
|
||||
"Cloud skills replace duplicate executor listings before budgeting; unique executor skills retain their descriptions and e1 aliases across turns.",
|
||||
&requests,
|
||||
&ContextSnapshotOptions::default().include_request_settings(),
|
||||
)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -128,8 +128,8 @@ struct StaticSkillProvider {
|
||||
main_prompt_contents: Option<String>,
|
||||
}
|
||||
|
||||
struct CatalogSkillProvider {
|
||||
catalog: SkillCatalog,
|
||||
pub(super) struct CatalogSkillProvider {
|
||||
pub(super) catalog: SkillCatalog,
|
||||
}
|
||||
|
||||
struct PausedCatalogSkillProvider {
|
||||
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
---
|
||||
source: core/tests/suite/scenarios_skill_catalog_dedup.rs
|
||||
expression: "context_snapshot::format_request_history_snapshot(\"Cloud skills replace duplicate executor listings before budgeting; unique executor skills retain their descriptions and e1 aliases across turns.\",\n&requests, &ContextSnapshotOptions::default().include_request_settings(),)"
|
||||
---
|
||||
Scenario: Cloud skills replace duplicate executor listings before budgeting; unique executor skills retain their descriptions and e1 aliases across turns.
|
||||
|
||||
## Window 1
|
||||
Settings:
|
||||
include: ["reasoning.encrypted_content"]
|
||||
model: "gpt-6-astra"
|
||||
parallel_tool_calls: false
|
||||
prompt_cache_key: "<PROMPT_CACHE_KEY 1>"
|
||||
reasoning: {"context":"all_turns","effort":"low"}
|
||||
store: false
|
||||
stream: true
|
||||
text: {"verbosity":"low"}
|
||||
tool_choice: "auto"
|
||||
-- request 1 (turn) --
|
||||
00:additional_tools/developer (3; hash=c803bbbd445f67cc):
|
||||
- namespace/functions; hash=48b2ce07cd26a39b
|
||||
- custom/exec
|
||||
- function/wait
|
||||
- function/request_user_input
|
||||
- function/request_user_input_async
|
||||
- namespace/clock: Tools for reading and waiting on time.; hash=f402e8c5e9b5e317
|
||||
- function/sleep
|
||||
- namespace/collaboration: Tools for spawning and managing sub-agents.; hash=fd994b11525da73c
|
||||
- function/followup_task
|
||||
- function/interrupt_agent
|
||||
- function/list_agents
|
||||
- function/send_message
|
||||
- function/spawn_agent
|
||||
- function/wait_agent
|
||||
01:message/developer:
|
||||
You are Codex, an agent based on GPT-6. You and the user share one workspace, and your job is to collabo...them until their intended goal is completely handled. [hash=efa50f074efa5803]
|
||||
|
||||
# When to ask the user for permission
|
||||
|
||||
Use your best judgement given task context for when you really need user permission, like a competent co...ork without ending the turn to clarify with the user. [hash=ca7367e658b51388]
|
||||
|
||||
User authorization and preferences persist across turns. Do not request permission again when the user h... any guidelines provided in skills or external files. [hash=911f91c92c72882c]
|
||||
|
||||
<OMITTED 155 LINES; ~4935 TOKENS; hash=5573869716192298>
|
||||
|
||||
- Skill naming: If a plugin contributes skills, those skill entries are prefixed with plugin_name: in the Skills list.
|
||||
- MCP naming: Plugin-provided MCP tools keep standard MCP identifiers such as mcp__server__tool; use tool provenance to tell which plugin they come from.
|
||||
- Trigger rules: If the user explicitly names a plugin, prefer capabilities associated with that plugin for that turn.
|
||||
- Relationship to capabilities: Plugins are not invoked directly. Use their underlying skills, MCP tools, and app tools to help solve the task.
|
||||
- Relevance: Determine what a plugin can help with from explicit user mention or from the plugin-associa..., MCP tools, and apps exposed elsewhere in this turn. [hash=620459a4e7b590f0]
|
||||
- Missing/blocked: If the user requests a plugin that does not have relevant callable capabilities for t..., say so briefly and continue with the best fallback. [hash=746b0469a8f6cb3e]
|
||||
|
||||
02:message/developer[4]:
|
||||
[01] <permissions instructions>
|
||||
Filesystem sandboxing defines which files can be read or written. `sandbox_mode` is `danger-full-access`...ll commands are permitted. Network access is enabled. [hash=c134061cebb94a1b]
|
||||
Approval policy is currently never. Do not provide the `sandbox_permissions` for any reason, commands will be rejected.
|
||||
</permissions instructions>
|
||||
[02] <collaboration_mode># Collaboration Mode: Default
|
||||
|
||||
You are now in Default mode. Any previous instructions for other modes (e.g. Plan mode) are no longer active.
|
||||
|
||||
Your active mode changes only when new developer instructions with a different `<collaboration_mode>...<...by themselves. Known mode names are Default and Plan. [hash=e75f6b12fc52d812]
|
||||
|
||||
## request_user_input availability
|
||||
|
||||
Use the `request_user_input` tool only when it is listed in the available tools for this turn.
|
||||
|
||||
Use the `request_user_input` tool only for optional questions where the answer would materially improve the quality of the work.
|
||||
|
||||
If `request_user_input` returns no answers, continue with best judgment instead of asking again or treating the turn as blocked.
|
||||
|
||||
Never use the `request_user_input` tool for permission requests or permission-related escalations.
|
||||
</collaboration_mode>
|
||||
[03] <skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...locators can be expanded using the skill roots table. [hash=f00d6a4d3e192f06]
|
||||
### Skill roots
|
||||
- `e0` = `skill://executor-a/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa` [hash=89039f2551ec08fb]
|
||||
- `e1` = `skill://executor-b/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb...bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb` [hash=aec7776011a3143f]
|
||||
Read a skill package directly with `skills.read({"package":"<package>"})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647]
|
||||
### Available skills
|
||||
- other:s0: Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidan...ce. UNIQUE_DESCRIPTION_END (executor package: e1/s0) [hash=1ce5d9766079d21b]
|
||||
- other:s1: Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidan...ce. UNIQUE_DESCRIPTION_END (executor package: e1/s1) [hash=d8432891813affeb]
|
||||
- other:s2: Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidance. Skill guidan...ce. UNIQUE_DESCRIPTION_END (executor package: e1/s2) [hash=1c05fa218dd21e8f]
|
||||
</skills_instructions>
|
||||
[04] <skills_instructions>
|
||||
## Skills
|
||||
A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that ...urce` locators use their provider's access mechanism. [hash=b6782e3abdba03b2]
|
||||
Read a skill package directly with `skills.read({"package":"<package>"})` to read its `SKILL.md`; root a...ackage is not provided, use `skills.list` to find it. [hash=25bf84ea2123d647]
|
||||
### Available skills
|
||||
- demo:s0: Cloud skill instructions. (cloud package: skill://cloud/s0)
|
||||
- demo:s1: Cloud skill instructions. (cloud package: skill://cloud/s1)
|
||||
- demo:s2: Cloud skill instructions. (cloud package: skill://cloud/s2)
|
||||
</skills_instructions>
|
||||
03:message/developer:
|
||||
<multi_agent_role>You are `/root`, the primary agent in a team of agents collaborating to fulfill the user's goals.
|
||||
|
||||
At the start of your turn, you are the active agent.
|
||||
You can spawn sub-agents to handle subtasks, and those sub-agents can spawn their own sub-agents.
|
||||
All agents in the team, including the agents that you can assign tasks to, are equally intelligent and capable, and have access to the same set of tools.
|
||||
|
||||
You can use `spawn_agent` to create a new agent, `followup_task` to give an existing agent a new task an...message to a running agent without triggering a turn. [hash=7d40ff8b58977202]
|
||||
`send_message` calls may be read by a human, so ensure they are legible. Always put proper spaces between words and/or numbers.
|
||||
<OMITTED 17 LINES; ~246 TOKENS; hash=780031e2df907944>
|
||||
- All agents use the same current working directory.
|
||||
- As a result, edits made by one agent are immediately visible to all other agents.
|
||||
|
||||
When calling `wait_agent`, prefer longer waits (minutes) to avoid busy polling.
|
||||
|
||||
There are 4 available concurrency slots, meaning that up to 4 agents can be active at once, including you.
|
||||
|
||||
Full-history forks (`fork_turns` omitted or `"all"`) inherit the parent model and reasoning effort and d...ne"` or a positive integer string.</multi_agent_role> [hash=c7f52e90688299ce]
|
||||
04:message/developer:
|
||||
<multi_agent_mode>Any earlier instruction enabling proactive multi-agent delegation no longer applies. D...elegation, or parallel agent work.</multi_agent_mode> [hash=2b6e037067b4390c]
|
||||
05:message/user:
|
||||
<environment_context>
|
||||
<current_date><CURRENT_DATE></current_date>
|
||||
<timezone><HOST_TIMEZONE></timezone>
|
||||
</environment_context>
|
||||
06:message/user:
|
||||
Which skills are available?
|
||||
-- request 2 (turn) --
|
||||
07:message/assistant:
|
||||
Skills are available.
|
||||
08:message/user:
|
||||
Are those same skills still available?
|
||||
Reference in New Issue
Block a user