openai/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-10-31 19:28:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
468b71f7a64c0dcafe8d44f3cafd18f75932b8a3
codex/codex-rs/ext/skills/tests/skills_extension.rs
T
Alex Daley 2edc8dcb0c Add an option to omit descriptions from skill catalogs (#57500) 
## What changed

Add the disabled-by-default `omit_skill_descriptions` feature flag to omit descriptions from model-visible host, executor, and cloud skill catalogs while retaining names and locators. Adapt catalog introductions, usage instructions, and alias budget accounting to the selected policy.

Persist the description policy in catalog snapshots and refresh usage instructions when it changes. Ignore description-only edits in catalog deltas while descriptions are omitted.

## Testing

Add coverage for alias token budgets, policy changes with older snapshots, and suppressed description-only updates. Extend scenario snapshots to cover plugin visibility and catalog deduplication with descriptions omitted.

GitOrigin-RevId: 5bf1dcee8c04059c7a78998affc1efcdef8ec883
2026-10-27 16:16:17 +00:00

3782 lines
140 KiB
Rust
Raw Blame History

use codex_protocol_test_support::ContentItemTestExt;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use assert_matches::assert_matches;
use codex_attachment_store::InlineAttachmentStore;
use codex_config::ConfigLayerEntry;
use codex_config::ConfigLayerSource;
use codex_config::ConfigLayerStack;
use codex_config::ConfigRequirementsToml;
use codex_exec_server::CapabilityRootDiscovery;
use codex_exec_server::DiscoverV2CapabilitiesResponse;
use codex_exec_server::EnvironmentManager;
use codex_exec_server::ExecutorCapabilityDiscoverySnapshot;
use codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot;
use codex_exec_server::ExecutorSkill;
use codex_exec_server::FileSystemEnvironmentAccessor;
use codex_exec_server::LOCAL_FS;
use codex_exec_server::SkillDiscoveryRoot;
use codex_extension_api::ConversationHistory;
use codex_extension_api::ExtensionData;
use codex_extension_api::ExtensionEventSink;
use codex_extension_api::ExtensionMetrics;
use codex_extension_api::ExtensionRegistryBuilder;
use codex_extension_api::ExtensionWarning;
use codex_extension_api::FunctionCallError;
use codex_extension_api::NoopTurnItemEmitter;
use codex_extension_api::PreviousWorldStateSection;
use codex_extension_api::RenderedWorldStateFragment;
use codex_extension_api::SelectedPluginSnapshot;
use codex_extension_api::SkillInvocationInput;
use codex_extension_api::SkillInvocationKind;
use codex_extension_api::ThreadStartInput;
use codex_extension_api::ToolCall;
use codex_extension_api::ToolCallSource;
use codex_extension_api::ToolPayload;
use codex_extension_api::TurnInputContext;
use codex_extension_api::TurnStartInput;
use codex_extension_api::WorldStateContributionInput;
use codex_extension_api::WorldStateSectionContribution;
use codex_models_manager::model_info::model_info_from_slug;
use codex_otel::MetricsClient;
use codex_otel::MetricsConfig;
use codex_otel::THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC;
use codex_otel::THREAD_SKILLS_ENABLED_TOTAL_METRIC;
use codex_otel::THREAD_SKILLS_KEPT_TOTAL_METRIC;
use codex_otel::THREAD_SKILLS_TRUNCATED_METRIC;
use codex_protocol::capabilities::CapabilityRootLocation;
use codex_protocol::capabilities::SelectedCapabilityRoot;
use codex_protocol::openai_models::ModelInfo;
use codex_protocol::protocol::EnvironmentConfigState;
use codex_protocol::protocol::Event;
use codex_protocol::protocol::SKILLS_INSTRUCTIONS_CLOSE_TAG;
use codex_protocol::protocol::SKILLS_INSTRUCTIONS_OPEN_TAG;
use codex_protocol::protocol::SessionSource;
use codex_protocol::protocol::SkillScope;
use codex_protocol::protocol::TruncationPolicy;
use codex_protocol::protocol::TurnEnvironmentSelection;
use codex_protocol::user_input::UserInput;
use codex_skills::SkillMetadata;
use codex_skills_extension::ExecutorSkillProvider;
use codex_skills_extension::HostSkillProvider;
use codex_skills_extension::HostSkillsLoadInput;
use codex_skills_extension::HostSkillsService;
use codex_skills_extension::HostSkillsSnapshot;
use codex_skills_extension::InjectedHostSkillPrompts;
use codex_skills_extension::SkillLoadOutcome;
use codex_skills_extension::SkillProviders;
use codex_skills_extension::SkillsExtensionConfig;
use codex_skills_extension::catalog::SkillAuthority;
use codex_skills_extension::catalog::SkillCatalog;
use codex_skills_extension::catalog::SkillCatalogEntry;
use codex_skills_extension::catalog::SkillPackageId;
use codex_skills_extension::catalog::SkillProviderError;
use codex_skills_extension::catalog::SkillReadResult;
use codex_skills_extension::catalog::SkillResourceId;
use codex_skills_extension::catalog::SkillSearchResult;
use codex_skills_extension::catalog::SkillSourceKind;
use codex_skills_extension::install;
use codex_skills_extension::install_with_providers;
use codex_skills_extension::install_with_providers_and_metrics;
use codex_skills_extension::provider::SkillListQuery;
use codex_skills_extension::provider::SkillProvider;
use codex_skills_extension::provider::SkillProviderFuture;
use codex_skills_extension::provider::SkillReadRequest;
use codex_skills_extension::provider::SkillSearchRequest;
use codex_utils_absolute_path::AbsolutePathBuf;
use codex_utils_path_uri::PathUri;
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
use opentelemetry_sdk::metrics::data::MetricData;
use pretty_assertions::assert_eq;
type TestResult = Result<(), Box<dyn std::error::Error>>;
#[path = "skills_extension/shadow_task_context_tests.rs"]
mod shadow_task_context_tests;
#[path = "skills_extension/presentation_dedup_tests.rs"]
mod presentation_dedup_tests;
#[path = "skills_extension/catalog_delta_tests.rs"]
mod catalog_delta_tests;
static NEXT_CODEX_HOME_ID: AtomicUsize = AtomicUsize::new(0);
const SKILLS_INTRO_WITH_ABSOLUTE_PATHS: &str = "A skill is a set of instructions provided through a `SKILL.md` source. Below is the list of skills that can be used. Each entry includes a name, description, and source locator. `file` locators are on the host filesystem, `executor package` locators are owned by their execution environment, `cloud package` locators are opaque package identifiers, and `custom resource` locators use their provider's access mechanism.";
const DEMO_SKILL_CONTENTS: &str =
"---\nname: demo\ndescription: Demo skill.\n---\n# Demo\n\nUse the demo skill.\n";
fn catalog_model_info() -> ModelInfo {
ModelInfo {
context_window: None,
max_context_window: None,
include_skills_usage_instructions: false,
..model_info_from_slug("test-model")
}
}
fn world_state_section<'a>(
sections: &'a [WorldStateSectionContribution],
id: &str,
) -> &'a WorldStateSectionContribution {
sections
.iter()
.find(|section| section.id() == id)
.unwrap_or_else(|| panic!("world-state section {id} should exist"))
}
async fn prepare_step(
registry: &codex_extension_api::ExtensionRegistry<TestConfig>,
input: codex_extension_api::StepPreparationInput<'_>,
) {
for contributor in registry.tool_contributors() {
contributor.prepare_for_step(input).await;
}
}
async fn prepared_world_state(
registry: &codex_extension_api::ExtensionRegistry<TestConfig>,
ready_selected_capability_roots: &[SelectedCapabilityRoot],
input: WorldStateContributionInput<'_>,
) -> Vec<WorldStateSectionContribution> {
let discovery = input
.step_store
.get::<codex_exec_server::ExecutorCapabilityDiscoverySnapshot>();
let snapshots = input
.step_store
.get::<Vec<codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot>>();
let selected_plugins = input
.step_store
.get::<SelectedPluginSnapshot>()
.unwrap_or_default();
prepare_step(
registry,
codex_extension_api::StepPreparationInput {
turn_id: input.turn_id,
ready_selected_capability_roots,
executor_capability_discovery: discovery.as_deref(),
executor_capability_discovery_v2: snapshots.as_deref().map(Vec::as_slice),
selected_plugins: &selected_plugins,
session_store: input.session_store,
thread_store: input.thread_store,
turn_store: input.turn_store,
step_store: input.step_store,
},
)
.await;
registry.context_contributors()[0]
.contribute_world_state(input)
.await
}
async fn prepared_turn_input(
registry: &codex_extension_api::ExtensionRegistry<TestConfig>,
input: TurnInputContext<'_>,
extension_metrics: Option<Arc<dyn codex_extension_api::ExtensionMetrics>>,
session_store: &ExtensionData,
thread_store: &ExtensionData,
turn_store: &ExtensionData,
) -> Vec<Box<dyn codex_extension_api::ContextualUserFragment + Send>> {
let roots = input
.step_store
.get::<Vec<codex_exec_server::ResolvedSelectedCapabilityRoot>>()
.map(|roots| {
roots
.iter()
.map(|root| root.selected_root().clone())
.collect::<Vec<_>>()
})
.unwrap_or_default();
let discovery = input
.step_store
.get::<codex_exec_server::ExecutorCapabilityDiscoverySnapshot>();
let snapshots = input
.step_store
.get::<Vec<codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot>>();
let selected_plugins = input
.step_store
.get::<SelectedPluginSnapshot>()
.unwrap_or_default();
prepare_step(
registry,
codex_extension_api::StepPreparationInput {
selected_plugins: &selected_plugins,
turn_id: &input.turn_id,
ready_selected_capability_roots: &roots,
executor_capability_discovery: discovery.as_deref(),
executor_capability_discovery_v2: snapshots.as_deref().map(Vec::as_slice),
session_store,
thread_store,
turn_store,
step_store: input.step_store,
},
)
.await;
registry.turn_input_contributors()[0]
.contribute(
input,
extension_metrics,
session_store,
thread_store,
turn_store,
)
.await
}
async fn skill_world_state_fragments(
registry: &codex_extension_api::ExtensionRegistry<TestConfig>,
session_store: &ExtensionData,
thread_store: &ExtensionData,
turn_id: &str,
) -> Result<(RenderedWorldStateFragment, RenderedWorldStateFragment), Box<dyn std::error::Error>> {
let turn_store = ExtensionData::new(turn_id);
let selected_roots = [SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills")?,
},
}];
let sections = prepared_world_state(
registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id,
environments: &[],
extension_metrics: None,
session_store,
thread_store,
turn_store: &turn_store,
},
)
.await;
let executor = world_state_section(&sections, "skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.ok_or("executor skills should render through world state")?;
let cloud = world_state_section(&sections, "cloud_skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.ok_or("cloud skills should render through world state")?;
Ok((executor, cloud))
}
async fn start_registered_turn(
registry: &codex_extension_api::ExtensionRegistry<TestConfig>,
session_store: &ExtensionData,
thread_store: &ExtensionData,
turn_id: &str,
) {
let turn_store = ExtensionData::new(turn_id);
let mode = codex_protocol::config_types::CollaborationMode {
mode: codex_protocol::config_types::ModeKind::Default,
settings: codex_protocol::config_types::Settings {
model: "test".to_string(),
reasoning_effort: None,
developer_instructions: None,
},
};
for contributor in registry.turn_lifecycle_contributors() {
contributor
.on_turn_start(TurnStartInput {
turn_id,
collaboration_mode: &mode,
token_usage_at_turn_start: None,
session_store,
thread_store,
turn_store: &turn_store,
})
.await;
}
}
#[tokio::test]
async fn installed_extension_uses_host_service_snapshot() -> TestResult {
let codex_home = test_codex_home();
let skill_path = codex_home.join("skills").join("demo").join("SKILL.md");
std::fs::create_dir_all(
skill_path
.parent()
.ok_or("skill path should have a parent")?,
)?;
std::fs::write(&skill_path, DEMO_SKILL_CONTENTS)?;
let mut config = default_config();
config.shadow_selection_enabled = true;
let mut builder = ExtensionRegistryBuilder::new();
install(&mut builder, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let skill_path = AbsolutePathBuf::try_from(skill_path)?;
let skill_path_string = skill_path.to_string_lossy().into_owned();
let mut outcome = SkillLoadOutcome::default();
outcome.skills.push(SkillMetadata {
name: "demo".to_string(),
description: "Demo skill.".to_string(),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: PathUri::from_abs_path(&skill_path),
scope: SkillScope::User,
plugin_id: None,
remote_plugin_id: None,
});
let loaded_skills = Arc::new(outcome);
let skill_prompt_path = skill_path_string.replace('\\', "/");
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::clone(&loaded_skills)));
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &ExtensionData::new("step"),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "$demo".to_string(),
text_elements: Vec::new(),
}],
environments: Vec::new(),
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
let expected_catalog = format!(
"{SKILLS_INSTRUCTIONS_OPEN_TAG}\n## Skills\n{SKILLS_INTRO_WITH_ABSOLUTE_PATHS}\n### Available skills\n- demo: Demo skill. (file: {skill_prompt_path})\n{SKILLS_INSTRUCTIONS_CLOSE_TAG}"
);
let expected_skill = format!(
"<skill>\n<name>demo</name>\n<path>{skill_prompt_path}</path>\n{DEMO_SKILL_CONTENTS}\n</skill>"
);
assert_eq!(
vec![("developer", expected_catalog), ("user", expected_skill),],
fragments
.iter()
.map(|fragment| (fragment.role(), fragment.render().expect_text()))
.collect::<Vec<_>>()
);
let injected_host_skill_prompts = turn_store
.get::<InjectedHostSkillPrompts>()
.ok_or("host skill prompt marker should be set")?;
assert!(injected_host_skill_prompts.contains_path(&skill_path_string));
std::fs::remove_dir_all(codex_home)?;
Ok(())
}
#[tokio::test]
async fn host_world_state_records_catalog_metrics_on_publish_and_change() -> TestResult {
let mut builder = ExtensionRegistryBuilder::new();
install(&mut builder, skills_extension_config);
let registry = builder.build();
let startup_metrics = Arc::new(RecordingMetrics::default());
let turn_metrics = Arc::new(RecordingMetrics::default());
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: Some(startup_metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let skill_path = AbsolutePathBuf::try_from(test_codex_home().join("skills/demo/SKILL.md"))?;
let mut outcome = SkillLoadOutcome::default();
outcome.skills.push(SkillMetadata {
name: "demo".to_string(),
description: "Demo skill.".to_string(),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: PathUri::from_abs_path(&skill_path),
scope: SkillScope::User,
plugin_id: None,
remote_plugin_id: None,
});
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome.clone())));
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: Some(turn_metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert_eq!(sections.len(), 2);
let host_section = world_state_section(&sections, "host_skills");
let (published_snapshot, fragment) =
host_section.render_diff(PreviousWorldStateSection::Absent);
let published_snapshot = published_snapshot.unwrap();
assert!(fragment.is_some());
let mut expected = expected_catalog_metric_samples("host_world_state", /*count*/ 1);
let expected_discovery = vec![expected_catalog_entries(
"executor", "complete", /*count*/ 0,
)];
assert_eq!(startup_metrics.samples(), expected_discovery);
assert_eq!(turn_metrics.samples(), expected);
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: Some(turn_metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert!(
world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Known(&published_snapshot))
.1
.is_none()
);
assert_eq!(startup_metrics.samples(), expected_discovery);
assert_eq!(turn_metrics.samples(), expected);
let second_skill_path =
AbsolutePathBuf::try_from(test_codex_home().join("skills/other/SKILL.md"))?;
outcome.skills.push(SkillMetadata {
name: "other".to_string(),
description: "Other skill.".to_string(),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: PathUri::from_abs_path(&second_skill_path),
scope: SkillScope::User,
plugin_id: None,
remote_plugin_id: None,
});
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: Some(turn_metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert!(
world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Known(&published_snapshot))
.1
.is_some()
);
expected.extend(expected_catalog_metric_samples(
"host_world_state",
/*count*/ 2,
));
assert_eq!(startup_metrics.samples(), expected_discovery);
assert_eq!(turn_metrics.samples(), expected);
Ok(())
}
#[tokio::test]
async fn persisted_host_snapshot_deduplicates_warning_after_reinitialization() -> TestResult {
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install(&mut builder, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let config = default_config();
let skill_path = AbsolutePathBuf::try_from(test_codex_home().join("skills/demo/SKILL.md"))?;
let mut outcome = SkillLoadOutcome::default();
outcome.skills.push(SkillMetadata {
name: "demo".to_string(),
description: "Demo skill.".to_string(),
short_description: None,
interface: None,
dependencies: None,
policy: None,
path_to_skills_md: PathUri::from_abs_path(&skill_path),
scope: SkillScope::User,
plugin_id: None,
remote_plugin_id: None,
});
let host_snapshot = HostSkillsSnapshot::new(Arc::new(outcome));
let thread_store = ExtensionData::new("thread");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let mut model_info = model_info_from_slug("test-model");
model_info.context_window = Some(50);
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(host_snapshot.clone());
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &model_info,
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
let host_section = world_state_section(&sections, "host_skills");
let (published_snapshot, fragment) =
host_section.render_diff(PreviousWorldStateSection::Absent);
let published_snapshot = published_snapshot.unwrap();
assert!(fragment.is_some());
event_rx.try_recv()?.into_warning();
assert!(event_rx.try_recv().is_err());
let resumed_thread_store = ExtensionData::new("thread");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &resumed_thread_store,
})
.await;
let mut model_info = model_info_from_slug("test-model");
model_info.context_window = Some(50);
let resumed_turn_store = ExtensionData::new("turn-2");
resumed_turn_store.insert(host_snapshot);
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &model_info,
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-2",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &resumed_thread_store,
turn_store: &resumed_turn_store,
},
)
.await;
assert!(
world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Known(&published_snapshot))
.1
.is_none()
);
assert!(event_rx.try_recv().is_err());
Ok(())
}
#[tokio::test]
async fn executor_cloud_and_host_share_catalog_world_state_flow() -> TestResult {
let provider = |mut entry: SkillCatalogEntry| {
if matches!(
entry.authority.kind,
SkillSourceKind::Cloud | SkillSourceKind::Host
) {
entry.name = "shared-skill".to_string();
}
Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![entry],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
})
};
let providers = SkillProviders::new()
.with_executor_provider(provider(test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/executor-skill",
"skill://executor/executor-skill/SKILL.md",
)))
.with_cloud_provider(provider(
test_entry(
SkillSourceKind::Cloud,
"codex_apps",
"cloud/cloud-skill",
"skill://cloud/cloud-skill/SKILL.md",
)
.with_remote_plugin_id(Some("shared-plugin".to_string())),
))
.with_host_provider(provider(
test_entry(
SkillSourceKind::Host,
"host",
"host/host-skill",
"/skills/host-skill/SKILL.md",
)
.with_display_path("/skills/host-skill/SKILL.md")
.with_remote_plugin_id(Some("shared-plugin".to_string())),
));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &default_config(),
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
// Host rendering stays separate even when its identity matches a Cloud skill.
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
let selected_roots = [SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills")?,
},
}];
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(
SkillLoadOutcome::default(),
)));
let metrics = Arc::new(RecordingMetrics::default());
let sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: Some(metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert_eq!(
sections
.iter()
.map(WorldStateSectionContribution::id)
.collect::<Vec<_>>(),
vec!["skills", "cloud_skills", "host_skills"]
);
assert!(metrics.samples().is_empty());
for (section_id, expected_line) in [
(
"skills",
"- executor-skill: Fix lint errors. (executor package: executor/executor-skill)",
),
(
"cloud_skills",
"- shared-skill: Fix lint errors. (cloud package: cloud/cloud-skill)",
),
(
"host_skills",
"- shared-skill: Fix lint errors. (file: /skills/host-skill/SKILL.md)",
),
] {
let fragment = world_state_section(&sections, section_id)
.render_diff(PreviousWorldStateSection::Absent)
.1
.ok_or("skill catalog should render through world state")?;
let body = fragment.body().clone().expect_text();
assert!(body.contains(expected_line));
}
let expected_metrics = [
"executor_world_state",
"cloud_world_state",
"host_world_state",
]
.into_iter()
.flat_map(|surface| expected_catalog_metric_samples(surface, /*count*/ 1))
.collect::<Vec<_>>();
assert_eq!(metrics.samples(), expected_metrics);
Ok(())
}
#[tokio::test]
async fn nonempty_executor_empty_host_records_catalog_metrics() -> TestResult {
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/lint-fix",
"lint-fix/SKILL.md",
)],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
});
let providers = SkillProviders::new()
.with_host_provider(Arc::new(HostSkillProvider::new()))
.with_executor_provider(executor_provider);
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let metrics = Arc::new(RecordingMetrics::default());
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let selected_roots = vec![SelectedCapabilityRoot {
id: "lint-fix".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills/lint-fix").expect("skill root URI"),
},
}];
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(
SkillLoadOutcome::default(),
)));
let sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: Some(metrics.clone()),
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert_eq!(sections.len(), 2);
assert!(
sections[0]
.render_diff(PreviousWorldStateSection::Absent)
.1
.is_some()
);
assert!(
world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.is_none()
);
let expected = expected_catalog_metric_samples("executor_world_state", /*count*/ 1);
assert_eq!(metrics.samples(), expected);
Ok(())
}
#[tokio::test]
async fn host_world_state_uses_provider_catalog_with_core_compatible_rendering() -> TestResult {
let list_calls = Arc::new(AtomicUsize::new(0));
let provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![
test_entry(
SkillSourceKind::Host,
"host",
"host/provider-skill",
"provider-skill/SKILL.md",
)
.with_short_description(Some("Short description.".to_string())),
],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: Some(Arc::clone(&list_calls)),
fail_first_list: false,
});
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(
&mut builder,
SkillProviders::new().with_host_provider(provider),
skills_extension_config,
);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(
SkillLoadOutcome::default(),
)));
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
let host_fragment = world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.ok_or("host provider catalog should render")?;
let body = host_fragment.body().clone().expect_text();
assert!(body.contains("Fix lint errors."));
assert!(!body.contains("Short description."));
assert_eq!(1, list_calls.load(Ordering::Relaxed));
Ok(())
}
#[tokio::test]
async fn shadow_selection_uses_host_catalog_when_instructions_are_disabled() -> TestResult {
let list_calls = Arc::new(AtomicUsize::new(0));
let provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![test_entry(
SkillSourceKind::Host,
"host",
"host/lint-fix",
"lint-fix/SKILL.md",
)],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: Some(Arc::clone(&list_calls)),
fail_first_list: false,
});
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"codex-skills-extension",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)?;
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers_and_metrics(
&mut builder,
SkillProviders::new().with_host_provider(provider),
Some(metrics.clone()),
skills_extension_config,
);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.include_instructions = false;
config.shadow_selection_enabled = true;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let turn_store = ExtensionData::new("turn-1");
let step_store = ExtensionData::new("step");
turn_store.insert(HostSkillsSnapshot::new(Arc::new(
SkillLoadOutcome::default(),
)));
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &step_store,
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "Fix lint errors.".to_string(),
text_elements: Vec::new(),
}],
environments: Vec::new(),
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
assert!(
world_state_section(&sections, "host_skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.is_none()
);
assert!(fragments.is_empty());
let snapshots = shadow_task_context_tests::collect_shadow_observations(
&metrics,
"codex.skills.shadow_selection.catalog_entries",
/*expected*/ 12,
)
.await?;
let catalog_entry_counts = snapshots
.iter()
.flat_map(opentelemetry_sdk::metrics::data::ResourceMetrics::scope_metrics)
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
.filter(|metric| metric.name() == "codex.skills.shadow_selection.catalog_entries")
.flat_map(|metric| match metric.data() {
AggregatedMetrics::F64(MetricData::Histogram(histogram)) => histogram
.data_points()
.map(opentelemetry_sdk::metrics::data::HistogramDataPoint::sum),
data => panic!("unexpected shadow catalog metric data: {data:?}"),
})
.collect::<Vec<_>>();
assert!(
catalog_entry_counts.iter().all(|count| *count == 1.0),
"every shadow selector should see the cached host skill: {catalog_entry_counts:?}"
);
assert_eq!(1, list_calls.load(Ordering::Relaxed));
Ok(())
}
#[tokio::test]
async fn shadow_lru_selector_recovers_a_skill_invoked_on_an_earlier_turn() -> TestResult {
let provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![test_entry(
SkillSourceKind::Host,
"host",
"host/lint-fix",
"lint-fix/SKILL.md",
)],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
});
let metrics = MetricsClient::new(
MetricsConfig::in_memory(
"test",
"codex-skills-extension",
env!("CARGO_PKG_VERSION"),
InMemoryMetricExporter::default(),
)
.with_runtime_reader(),
)?;
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers_and_metrics(
&mut builder,
SkillProviders::new().with_host_provider(provider),
Some(metrics.clone()),
skills_extension_config,
);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.include_instructions = false;
config.shadow_selection_enabled = true;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
for (turn_id, text) in [("turn-1", "Fix lint errors."), ("turn-2", "continue")] {
let turn_store = ExtensionData::new(turn_id);
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &ExtensionData::new("step"),
turn_id: turn_id.to_string(),
user_input: vec![UserInput::Text {
text: text.to_string(),
text_elements: Vec::new(),
}],
environments: vec![codex_extension_api::TurnInputEnvironment {
environment_id: "test".to_string(),
cwd: PathUri::from_host_native_path(std::env::current_dir().expect("test cwd"))
.expect("absolute cwd"),
is_primary: true,
fs: &FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS),
}],
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
assert!(fragments.is_empty());
registry.skill_invocation_contributors()[0]
.on_skill_invocation(SkillInvocationInput {
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
turn_id,
skill_resource: "lint-fix/SKILL.md",
kind: SkillInvocationKind::Implicit,
})
.await;
}
let snapshots = shadow_task_context_tests::collect_shadow_observations(
&metrics,
"codex.skills.shadow_selection.invocation",
/*expected*/ 24,
)
.await?;
let selector_hits = snapshots
.iter()
.flat_map(opentelemetry_sdk::metrics::data::ResourceMetrics::scope_metrics)
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
.filter(|metric| metric.name() == "codex.skills.shadow_selection.invocation")
.flat_map(|metric| match metric.data() {
AggregatedMetrics::U64(MetricData::Sum(sum)) => sum.data_points(),
data => panic!("unexpected shadow invocation metric data: {data:?}"),
})
.filter_map(|point| {
let method = point
.attributes()
.find(|attribute| attribute.key.as_str() == "method")?
.value
.as_str();
if !matches!(
method.as_ref(),
"lru_v1"
| "lru_plus_lexical_v1"
| "lru_plus_character_routing_v1"
| "lru_plus_lexical_character_routing_v1"
) {
return None;
}
let hit = point
.attributes()
.find(|attribute| attribute.key.as_str() == "hit")?
.value
.as_str()
.to_string();
Some((method.to_string(), hit, point.value()))
})
.fold(
std::collections::BTreeMap::new(),
|mut totals, (method, hit, count)| {
*totals.entry((method, hit)).or_insert(0) += count;
totals
},
)
.into_iter()
.map(|((method, hit), count)| (method, hit, count))
.collect::<Vec<_>>();
assert_eq!(
vec![
(
"lru_plus_character_routing_v1".to_string(),
"true".to_string(),
2,
),
(
"lru_plus_lexical_character_routing_v1".to_string(),
"true".to_string(),
2,
),
("lru_plus_lexical_v1".to_string(), "true".to_string(), 2),
("lru_v1".to_string(), "false".to_string(), 1),
("lru_v1".to_string(), "true".to_string(), 1),
],
selector_hits
);
Ok(())
}
#[tokio::test]
async fn selected_executor_catalog_caches_root_selections_per_turn_and_reuses_discovery_snapshots()
-> TestResult {
let step_store = ExtensionData::new("step-1");
let read_requests = Arc::new(Mutex::new(Vec::new()));
let list_calls = Arc::new(AtomicUsize::new(0));
let resource = SkillResourceId::environment(
"lint-fix/SKILL.md",
"env-1",
PathUri::parse("file:///skills/lint-fix/SKILL.md")?,
);
let mut entry = test_entry(
SkillSourceKind::Executor,
"env-1",
"executor/lint-fix",
"lint-fix/SKILL.md",
);
entry.main_prompt = resource.clone();
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![entry],
warnings: Vec::new(),
},
read_requests: Arc::clone(&read_requests),
list_calls: Some(Arc::clone(&list_calls)),
fail_first_list: false,
});
let providers = SkillProviders::new().with_executor_provider(executor_provider);
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let selected_roots = vec![SelectedCapabilityRoot {
id: "lint-fix".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills/lint-fix").expect("skill root URI"),
},
}];
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let prompt_fragments = registry.context_contributors()[0]
.contribute_thread_context(&session_store, &thread_store)
.await;
assert!(prompt_fragments.is_empty());
let turn_store = ExtensionData::new("turn-1");
let turn_environment = TurnEnvironmentSelection {
selected_capability_roots: Default::default(),
environment_id: "env-1".to_string(),
cwd: PathUri::parse("file:///workspace").expect("cwd URI"),
workspace_roots: Vec::new(),
config: EnvironmentConfigState::FromThread,
};
let available_sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: std::slice::from_ref(&turn_environment),
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert_eq!(1, available_sections.len());
let (available_snapshot, available_fragment) =
available_sections[0].render_diff(PreviousWorldStateSection::Absent);
let available_snapshot = available_snapshot.unwrap();
let available_fragment = available_fragment.ok_or("available skills should render")?;
let available_body = available_fragment.body().clone().expect_text();
assert!(available_body.contains("lint-fix"));
assert!(available_body.contains("(executor package: executor/lint-fix)"));
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &step_store,
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "$lint-fix please".to_string(),
text_elements: Vec::new(),
}],
environments: vec![codex_extension_api::TurnInputEnvironment {
environment_id: turn_environment.environment_id.clone(),
cwd: turn_environment.cwd.clone(),
is_primary: true,
fs: &FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS),
}],
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
assert_eq!(1, fragments.len());
assert_eq!("user", fragments[0].role());
let rendered = fragments[0].render().expect_text();
assert!(rendered.contains("<name>lint-fix</name>"));
assert!(rendered.contains("# Lint Fix"));
assert_eq!(
vec![(
SkillAuthority::new(SkillSourceKind::Executor, "env-1"),
SkillPackageId("executor/lint-fix".to_string()),
resource,
)],
read_request_keys(&read_requests)
);
let available_world_state =
serde_json::Map::from_iter([("skills".to_string(), available_snapshot.clone())]);
let unavailable_turn_store = ExtensionData::new("turn-2");
let unavailable_sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: Some(&available_world_state),
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-2",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &unavailable_turn_store,
},
)
.await;
let (unavailable_snapshot, unavailable_fragment) =
unavailable_sections[0].render_diff(PreviousWorldStateSection::Known(&available_snapshot));
let unavailable_snapshot = unavailable_snapshot.unwrap();
assert_eq!(
unavailable_snapshot,
serde_json::json!({
"body": null,
"includeInstructions": true,
"lastAvailableFingerprint": blake3::hash(
available_snapshot["body"].as_str().ok_or("available catalog")?.as_bytes()
).to_hex().to_string(),
}),
);
let unavailable_fragment = unavailable_fragment.ok_or("removed skills should render")?;
let unavailable_body = unavailable_fragment.body().clone().expect_text();
assert!(unavailable_body.contains("No selected-environment skills"));
let unavailable_world_state =
serde_json::Map::from_iter([("skills".to_string(), unavailable_snapshot.clone())]);
let restored_turn_store = ExtensionData::new("turn-3");
let restored_sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: Some(&unavailable_world_state),
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-3",
environments: &[turn_environment],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &restored_turn_store,
},
)
.await;
let (restored_snapshot, restored_fragment) =
restored_sections[0].render_diff(PreviousWorldStateSection::Known(&unavailable_snapshot));
let restored_snapshot = restored_snapshot.unwrap();
let restored_fragment = restored_fragment.ok_or("restored skills should render")?;
assert_eq!(
restored_fragment.body().clone().expect_text(),
"\n## Skills update\nThe previously listed selected-environment skills are available again.\n",
);
assert!(restored_sections[0].matches_retained_fragment("developer", &available_body));
assert!(!restored_sections[0].matches_retained_fragment("developer", &unavailable_body));
assert_eq!(
restored_sections[0]
.render_diff(PreviousWorldStateSection::Absent)
.1,
Some(available_fragment),
);
assert_eq!(2, list_calls.load(Ordering::Relaxed));
// Retained steps keep their catalog even after the selected roots change.
let retained_sections = registry.context_contributors()[0]
.contribute_world_state(WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
})
.await;
assert_eq!(
retained_sections[0]
.render_diff(PreviousWorldStateSection::Absent)
.0
.as_ref(),
Some(&available_snapshot)
);
assert_eq!(2, list_calls.load(Ordering::Relaxed));
let failed_discovery = ExecutorCapabilityDiscoverySnapshot::new(
&selected_roots,
vec![Err("exec-server transport disconnected".to_string())],
Default::default(),
);
let recovered_discovery = ExecutorCapabilityDiscoverySnapshot::new(
&selected_roots,
vec![Ok(Arc::new(CapabilityRootDiscovery {
id: "lint-fix".to_string(),
path: PathUri::parse("file:///skills/lint-fix")?,
plugin: None,
skills: Vec::new(),
namespace_manifests: Vec::new(),
warnings: Vec::new(),
error: None,
}))],
Default::default(),
);
for (turn_id, discovery, expected_list_calls) in [
("failed-discovery", &failed_discovery, 3),
("recovered-discovery", &recovered_discovery, 4),
("cached-discovery", &recovered_discovery, 4),
] {
let step_store = ExtensionData::new("step");
step_store.insert(discovery.clone());
prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id,
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &ExtensionData::new(turn_id),
},
)
.await;
assert_eq!(expected_list_calls, list_calls.load(Ordering::Relaxed));
}
let mut listing_disabled_config = config.clone();
listing_disabled_config.include_instructions = false;
registry.config_contributors()[0].on_config_changed(
&session_store,
&thread_store,
&config,
&listing_disabled_config,
);
let listing_disabled_turn_store = ExtensionData::new("turn-4");
let listing_disabled_sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-4",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &listing_disabled_turn_store,
},
)
.await;
let (listing_disabled_snapshot, listing_disabled_fragment) = listing_disabled_sections[0]
.render_diff(PreviousWorldStateSection::Known(&restored_snapshot));
let listing_disabled_fragment =
listing_disabled_fragment.ok_or("disabled skill listing should render")?;
assert_eq!(
"\n## Skills update\nSelected-environment skills are not listed automatically. Explicit skill mentions can still be resolved when available.\n",
listing_disabled_fragment.body().clone().expect_text()
);
let mut normalized_listing_disabled_snapshot = listing_disabled_snapshot.unwrap();
normalized_listing_disabled_snapshot
.as_object_mut()
.ok_or("skills snapshot should be an object")?
.remove("body");
assert!(
listing_disabled_sections[0]
.render_diff(PreviousWorldStateSection::Known(
&normalized_listing_disabled_snapshot
))
.1
.is_none()
);
let first_root = selected_roots[0].clone();
let second_root = SelectedCapabilityRoot {
id: "second-root".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills/second-root")?,
},
};
let cache_environment = Arc::new(codex_exec_server::Environment::create_for_tests(
/*exec_server_url*/ None,
)?);
let cache_manager = EnvironmentManager::default_for_tests();
let previous_list_calls = list_calls.load(Ordering::Relaxed);
let mut current_config = listing_disabled_config;
for (turn_id, roots, include_bundled_skills, expected_new_calls) in [
("cache-turn-1", vec![first_root.clone()], true, 1),
("cache-turn-1", vec![first_root.clone()], true, 1),
(
"cache-turn-1",
vec![first_root.clone(), second_root.clone()],
true,
2,
),
(
"cache-turn-1",
vec![second_root.clone(), first_root.clone()],
true,
3,
),
("cache-turn-1", vec![first_root.clone()], true, 4),
("cache-turn-2", vec![first_root.clone()], true, 5),
("cache-turn-2", vec![first_root.clone()], false, 6),
("cache-turn-2", vec![first_root], false, 6),
] {
let mut next_config = current_config.clone();
next_config.bundled_skills_enabled = include_bundled_skills;
registry.config_contributors()[0].on_config_changed(
&session_store,
&thread_store,
&current_config,
&next_config,
);
current_config = next_config;
let cache_step = ExtensionData::new("cache-step");
let captured = HashMap::from([("env-1".to_string(), Some(Arc::clone(&cache_environment)))]);
cache_step.insert(
cache_manager
.resolve_selected_capability_roots(&roots, &captured)
.await,
);
prepared_world_state(
&registry,
&roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &cache_step,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id,
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &ExtensionData::new(turn_id),
},
)
.await;
assert_eq!(
previous_list_calls + expected_new_calls,
list_calls.load(Ordering::Relaxed)
);
}
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn overlapping_executor_catalogs_render_their_own_roots() -> TestResult {
use tracing::Subscriber;
use tracing::instrument::WithSubscriber;
use tracing_subscriber::Layer;
use tracing_subscriber::layer::Context;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::registry::LookupSpan;
// Pause after A's refresh returns, before World State can read the catalog. Pausing the
// provider would be too early: A would still publish its result after B finishes.
struct PauseAfterRefresh {
entered: Mutex<Option<tokio::sync::oneshot::Sender<()>>>,
resume: Mutex<std::sync::mpsc::Receiver<()>>,
}
impl<S: Subscriber + for<'a> LookupSpan<'a>> Layer<S> for PauseAfterRefresh {
fn on_close(&self, id: tracing::span::Id, context: Context<'_, S>) {
if context
.span(&id)
.is_some_and(|span| span.name() == "skills.executor.refresh_executor_catalog")
&& let Some(entered) = self.entered.lock().unwrap().take()
{
entered.send(()).unwrap();
self.resume
.lock()
.unwrap()
.recv_timeout(std::time::Duration::from_secs(/*secs*/ 10))
.expect("B should finish before A reads its catalog");
}
}
}
let temp = tempfile::tempdir()?;
let root = |name: &str| -> Result<SelectedCapabilityRoot, Box<dyn std::error::Error>> {
let path = temp.path().join(name);
std::fs::create_dir_all(path.join(name))?;
std::fs::write(
path.join(name).join("SKILL.md"),
DEMO_SKILL_CONTENTS.replace("demo", name),
)?;
Ok(SelectedCapabilityRoot {
id: name.to_string(),
location: CapabilityRootLocation::Environment {
environment_id: codex_exec_server::LOCAL_ENVIRONMENT_ID.to_string(),
path: PathUri::from_host_native_path(path)?,
},
})
};
let root_a = root("skill-a")?;
let root_b = root("skill-b")?;
let mut builder = ExtensionRegistryBuilder::new();
let provider = codex_skills_extension::ExecutorSkillProvider::new_with_restriction_product(
Arc::new(codex_exec_server::EnvironmentManager::default_for_tests()),
/*restriction_product*/ None,
);
install_with_providers(
&mut builder,
SkillProviders::new().with_executor_provider(Arc::new(provider)),
skills_extension_config,
);
let registry = Arc::new(builder.build());
let session_store = Arc::new(ExtensionData::new("session"));
let thread_store = Arc::new(ExtensionData::new("thread"));
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &default_config(),
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let thread_id = codex_protocol::ThreadId::new();
let render = |root: SelectedCapabilityRoot| {
let registry = Arc::clone(&registry);
let session_store = Arc::clone(&session_store);
let thread_store = Arc::clone(&thread_store);
async move {
let step_store = ExtensionData::new(&root.id);
let sections = prepared_world_state(
&registry,
&[root],
WorldStateContributionInput {
model_info: &catalog_model_info(),
thread_id,
turn_id: "turn",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &step_store,
step_store: &step_store,
previous_world_state: None,
},
)
.await;
world_state_section(&sections, "skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.expect("executor skills should render")
.body()
.clone()
.expect_text()
}
};
let (entered, paused) = tokio::sync::oneshot::channel();
let (release, resume) = std::sync::mpsc::channel();
// Keep tracing aware of uninstrumented threads so a parallel test cannot register
// the refresh callsite as globally disabled before this subscriber reaches it.
let _interest_cache_guard =
tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
let subscriber = tracing_subscriber::registry().with(
PauseAfterRefresh {
entered: Mutex::new(Some(entered)),
resume: Mutex::new(resume),
}
.with_filter(tracing_subscriber::filter::filter_fn(|metadata| {
metadata.name() == "skills.executor.refresh_executor_catalog"
})),
);
let a = tokio::spawn(render(root_a).with_subscriber(subscriber));
tokio::time::timeout(std::time::Duration::from_secs(/*secs*/ 10), paused).await??;
let b = render(root_b).await;
release.send(())?;
let a = a.await?;
assert!(a.contains("skill-a") && !a.contains("skill-b"), "{a}");
assert!(b.contains("skill-b") && !b.contains("skill-a"), "{b}");
Ok(())
}
#[tokio::test]
async fn default_context_truncates_catalog_descriptions() -> TestResult {
let description = "x".repeat(1_025);
// A small cloud catalog leaves its unused allowance to filesystem descriptions.
let executor_entries = (0..5)
.map(|index| {
let package_id = format!("executor/executor-long-description-{index}");
let mut entry = test_entry(
SkillSourceKind::Executor,
"env-1",
&package_id,
&format!("skill://{package_id}/SKILL.md"),
);
entry.description = description.clone();
entry
})
.collect();
let mut cloud_entry = test_entry(
SkillSourceKind::Cloud,
"codex_apps",
"cloud/cloud-long-description",
"skill://cloud/cloud-long-description/SKILL.md",
);
cloud_entry.description = description.clone();
let providers = SkillProviders::new()
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: executor_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}))
.with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![cloud_entry],
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
let (executor, cloud) =
skill_world_state_fragments(&registry, &session_store, &thread_store, "turn-1").await?;
let executor = executor.body().clone().expect_text();
let cloud = cloud.body().clone().expect_text();
let expected_description = "x".repeat(1_021) + "...";
for index in 0..5 {
assert!(executor.contains(&format!(
"- executor-long-description-{index}: {expected_description} (executor package:"
)));
}
assert!(cloud.contains("- cloud-long-description:"));
for rendered in [executor, cloud] {
assert!(rendered.contains(&expected_description));
assert!(!rendered.contains(&"x".repeat(1_024)));
assert!(!rendered.contains(&description));
}
Ok(())
}
#[tokio::test]
async fn catalog_rebalances_only_to_avoid_omissions_and_retains_the_allocation() -> TestResult {
enum Phase {
Initial,
FirstReady,
Disconnected,
Reconnected,
Resumed,
CloudCatalogGrew,
BudgetChanged,
}
// Description-only pressure, filesystem omissions that can be rescued, and
// unavoidable omissions. Check both token and fallback character accounting.
for (executor_count, cloud_count, rebalance, all_fit) in [
(17, 82, false, true),
(40, 5, true, true),
(40, 160, false, false),
] {
for context_window in [None, Some(100_000)] {
let description = "x".repeat(1_025);
let executor_entries: Vec<_> = (0..executor_count)
.map(|index| {
let package_id = format!("executor/executor-skill-{index:02}");
let mut entry = test_entry(
SkillSourceKind::Executor,
"env-1",
&package_id,
&format!("skill://{package_id}/SKILL.md"),
);
entry.description = description.clone();
entry
})
.collect();
let registry_for_cloud_count = |cloud_count| {
let cloud_entries = (0..cloud_count)
.map(|index| {
let package_id = format!("cloud/cloud-skill-{index:02}");
let mut entry = test_entry(
SkillSourceKind::Cloud,
"codex_apps",
&package_id,
&format!("skill://{package_id}/SKILL.md"),
);
entry.description = description.clone();
entry
})
.collect();
let providers = SkillProviders::new()
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: executor_entries.clone(),
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}))
.with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: cloud_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
builder.build()
};
let mut registry = registry_for_cloud_count(cloud_count);
let selected_roots = [SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills")?,
},
}];
let mut model = ModelInfo {
context_window,
..catalog_model_info()
};
let session_store = ExtensionData::new("session");
let mut thread_store = ExtensionData::new("thread");
let mut previous = serde_json::Map::new();
let mut initial_cloud = None;
let mut accepted_cloud = None;
let mut phases = vec![
Phase::Initial,
Phase::FirstReady,
Phase::Disconnected,
Phase::Reconnected,
Phase::Disconnected,
Phase::Reconnected,
Phase::Resumed,
];
if rebalance {
phases.extend([
Phase::CloudCatalogGrew,
Phase::Disconnected,
Phase::Reconnected,
Phase::Resumed,
]);
}
phases.push(Phase::BudgetChanged);
for (step, phase) in phases.into_iter().enumerate() {
if matches!(phase, Phase::CloudCatalogGrew) {
// Refresh the cloud inventory on resume while retaining the saved allocation.
registry = registry_for_cloud_count(/*cloud_count*/ 15);
}
if matches!(
phase,
Phase::Initial | Phase::Resumed | Phase::CloudCatalogGrew
) {
thread_store = ExtensionData::new("thread");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &default_config(),
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
}
if matches!(phase, Phase::BudgetChanged) {
model.context_window = Some(200_000);
}
let ready = matches!(
phase,
Phase::FirstReady
| Phase::Reconnected
| Phase::Resumed
| Phase::CloudCatalogGrew
);
let turn_id = format!("turn-{step}");
let turn_store = ExtensionData::new(&turn_id);
start_registered_turn(&registry, &session_store, &thread_store, &turn_id).await;
let sections = prepared_world_state(
&registry,
if ready { &selected_roots } else { &[] },
WorldStateContributionInput {
previous_world_state: Some(&previous),
model_info: &model,
thread_id: codex_protocol::ThreadId::new(),
turn_id: &turn_id,
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
step_store: &turn_store,
},
)
.await;
let rendered = sections
.iter()
.map(|section| {
let prior = previous.get(section.id()).map_or(
PreviousWorldStateSection::Absent,
PreviousWorldStateSection::Known,
);
(section.id().to_string(), section.render_diff(prior))
})
.collect::<std::collections::BTreeMap<_, _>>();
let cloud = &rendered["cloud_skills"];
let cloud_snapshot = cloud.0.as_ref().expect("cloud snapshot");
let cloud_body = cloud_snapshot["body"].as_str().ok_or("cloud catalog")?;
match phase {
Phase::Initial => initial_cloud = Some(cloud_body.to_string()),
Phase::FirstReady => {
assert_eq!(initial_cloud.as_deref() != Some(cloud_body), rebalance);
accepted_cloud = Some(cloud_snapshot.clone());
let executor = rendered["skills"].0.as_ref().expect("executor snapshot");
let executor_body = executor["body"].as_str().ok_or("executor catalog")?;
assert_eq!(
executor_body
.lines()
.filter(|line| line.starts_with("- executor-skill-"))
.count()
== executor_count,
all_fit
);
if all_fit {
assert_eq!(
cloud_body
.lines()
.filter(|line| line.starts_with("- cloud-skill-"))
.count(),
cloud_count
);
}
}
Phase::CloudCatalogGrew => {
let allocation = &cloud_snapshot["allocation"];
let previous_limit = previous["cloud_skills"]["allocation"]["cloudLimit"]
.as_u64()
.ok_or("previous cloud limit")?;
assert!(
allocation["cloudLimit"].as_u64().ok_or("cloud limit")?
> previous_limit
);
assert_ne!(
allocation["cloudCatalogFingerprint"],
previous["cloud_skills"]["allocation"]["cloudCatalogFingerprint"]
);
assert_eq!(
cloud_body
.lines()
.filter(|line| line.starts_with("- cloud-skill-"))
.count(),
15
);
let executor = rendered["skills"].0.as_ref().expect("executor snapshot");
assert_eq!(
executor["body"]
.as_str()
.ok_or("executor catalog")?
.lines()
.filter(|line| line.starts_with("- executor-skill-"))
.count(),
executor_count
);
accepted_cloud = Some(cloud_snapshot.clone());
}
Phase::Disconnected | Phase::Reconnected | Phase::Resumed => {
assert_eq!(Some(cloud_snapshot), accepted_cloud.as_ref());
assert!(cloud.1.is_none());
}
Phase::BudgetChanged => {
// A genuinely different total budget starts a new allocation.
assert_eq!(
cloud_snapshot["allocation"],
serde_json::json!({
"totalBudget": {"tokens": 4_000}, "cloudLimit": 3_000,
"cloudCatalogFingerprint": previous["cloud_skills"]["allocation"]["cloudCatalogFingerprint"],
})
);
}
}
previous = rendered
.into_iter()
.map(|(id, (snapshot, _))| (id, snapshot.expect("skills snapshot")))
.collect();
}
}
}
Ok(())
}
#[tokio::test]
async fn extreme_budget_pressure_removes_descriptions_before_omitting_entries() -> TestResult {
let executor_entries = (0..40)
.map(|index| {
let package_id = format!("executor/executor-skill-{index:03}");
let mut entry = test_entry(
SkillSourceKind::Executor,
"env-1",
&package_id,
&format!("skill://{package_id}/SKILL.md"),
);
entry.description = format!("executor-description-{index:03}");
entry
})
.collect();
let cloud_entries = (0..160)
.map(|index| {
let package_id = format!("cloud/cloud-skill-{index:03}");
let mut entry = test_entry(
SkillSourceKind::Cloud,
"codex_apps",
&package_id,
&format!("skill://{package_id}/SKILL.md"),
);
entry.description = format!("cloud-description-{index:03}");
entry
})
.collect();
let providers = SkillProviders::new()
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: executor_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}))
.with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: cloud_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
let (executor, cloud) =
skill_world_state_fragments(&registry, &session_store, &thread_store, "turn-1").await?;
let executor = executor.body().clone().expect_text();
let cloud = cloud.body().clone().expect_text();
let included_executor_count = executor
.lines()
.filter(|line| line.starts_with("- executor-skill-"))
.count();
let included_cloud_count = cloud
.lines()
.filter(|line| line.starts_with("- cloud-skill-"))
.count();
assert!(included_executor_count > 0);
assert!(included_executor_count < 40);
assert!(included_cloud_count > 0);
assert!(included_cloud_count < 160);
assert!(executor.contains("- executor-skill-000: (executor package:"));
assert!(cloud.contains("- cloud-skill-000: (cloud package:"));
assert!(!cloud.contains("- cloud-skill-159:"));
for (total, included, rendered) in [
(40, included_executor_count, executor),
(160, included_cloud_count, cloud),
] {
assert!(!rendered.contains("description-"));
assert!(rendered.contains("additional skills omitted from this bounded skills list"));
let omitted_count = total - included;
let warning = event_rx.try_recv()?.into_warning();
assert_eq!(warning.thread_id, "thread");
assert_eq!(warning.turn_id.as_deref(), Some("turn-1"));
assert_eq!(
warning.message,
format!(
"Exceeded skills context budget. All skill descriptions were removed and {omitted_count} additional skills were not included in the model-visible skills list."
)
);
}
assert!(event_rx.try_recv().is_err());
Ok(())
}
#[tokio::test]
async fn skills_list_only_returns_model_visible_bounded_metadata() -> TestResult {
const OVERSIZED_WARNING: &str = "Some skills were omitted because their metadata is too large.";
const PROVIDER_WARNING: &str = "Some cloud skills could not be discovered because the provider disconnected before returning the complete catalog.";
let supplementary_warning = "w".repeat(256);
let opaque_suffix = "\\".repeat(1_500);
let mut entry = test_entry(
SkillSourceKind::Cloud,
"codex_apps",
&format!("cloud/{opaque_suffix}"),
&format!("skill://cloud/{opaque_suffix}/SKILL.md"),
);
entry.description = "x".repeat(1_025);
let [first_entry, mut middle_entry, mut final_entry] = ["p0", "p1", "p2"].map(|name| {
test_entry(
SkillSourceKind::Cloud,
"codex_apps",
name,
&format!("skill://{name}/SKILL.md"),
)
});
middle_entry.description = "d".repeat(110);
final_entry.description = "d".repeat(50);
let providers = SkillProviders::new().with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![
entry,
first_entry,
middle_entry,
final_entry,
test_entry(
SkillSourceKind::Cloud,
"codex_apps",
"cloud/hidden",
"skill://cloud/hidden/SKILL.md",
)
.hidden_from_prompt(),
],
warnings: vec![
PROVIDER_WARNING.to_string(),
supplementary_warning.clone(),
supplementary_warning.clone(),
supplementary_warning.clone(),
],
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
let step_store = ExtensionData::new("step");
prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &ExtensionData::new("turn-1"),
},
)
.await;
let tools =
registry.tool_contributors()[0].tools_for_step(&session_store, &thread_store, &step_store);
let list_tool = tools
.iter()
.find(|tool| tool.tool_name().name == "list")
.ok_or("skills.list tool should be registered")?;
let payload = ToolPayload::Function {
arguments: serde_json::json!({"authority": {"kind": "cloud"}}).to_string(),
};
let call = ToolCall {
turn_id: "turn-1".to_string(),
call_id: "call-1".to_string(),
tool_name: list_tool.tool_name(),
model: "gpt-test".to_string(),
codex_turn_metadata: None,
truncation_policy: TruncationPolicy::Bytes(10_000),
source: ToolCallSource::Direct,
conversation_history: ConversationHistory::default(),
turn_item_emitter: Arc::new(NoopTurnItemEmitter),
attachment_store: Arc::new(InlineAttachmentStore),
ephemeral: false,
environments: Vec::new(),
payload: payload.clone(),
};
let output = list_tool.handle(call.clone()).await?;
let complete_response = output
.post_tool_use_response("call-1", &payload)
.ok_or("skills.list should expose structured output")?;
let complete_skills = complete_response["skills"]
.as_array()
.ok_or("skills.list should return skills")?;
assert_eq!(complete_skills.len(), 4);
assert_eq!(complete_skills[0]["description"], "x".repeat(1_021) + "...");
assert_eq!(
complete_response["warnings"].as_array().map(Vec::len),
Some(4)
);
assert_eq!(complete_response["next_cursor"], serde_json::Value::Null);
let mut cursor = None;
for (
index,
(truncation_limit, byte_budget, expected_indices, expected_warnings, cursor_offset),
) in [
// The escaped entry fits alone, but not alongside even one provider warning.
(6_458, 7_750, vec![], vec![PROVIDER_WARNING], Some("0")),
(6_458, 7_750, vec![0], vec![], Some("1")),
(266, 320, vec![1], vec![], Some("2")),
// A smaller budget omits the middle entry and must report the new omission.
(183, 220, vec![], vec![OVERSIZED_WARNING], Some("2")),
(183, 220, vec![3], vec![], None),
// Neither report may hide the other when they need separate pages.
(183, 220, vec![], vec![PROVIDER_WARNING], Some("0")),
(183, 220, vec![], vec![OVERSIZED_WARNING], Some("0")),
// The last retained entry fits without a cursor, despite an oversized suffix.
(151, 182, vec![], vec![OVERSIZED_WARNING], Some("0")),
(151, 182, vec![1], vec![], None),
// An omission notice must not displace an affordable provider warning.
(
2_000,
2_400,
vec![1, 2, 3],
vec![
PROVIDER_WARNING,
&supplementary_warning,
&supplementary_warning,
&supplementary_warning,
OVERSIZED_WARNING,
],
None,
),
]
.into_iter()
.enumerate()
{
let payload = ToolPayload::Function {
arguments: serde_json::json!({
"authority": {"kind": "cloud"},
"cursor": cursor,
})
.to_string(),
};
let call_id = format!("bounded-page-{index}");
let output = list_tool
.handle(ToolCall {
call_id: call_id.clone(),
truncation_policy: TruncationPolicy::Bytes(truncation_limit),
payload: payload.clone(),
..call.clone()
})
.await?;
assert!(output.contains_external_context());
let response = output
.post_tool_use_response(&call_id, &payload)
.ok_or("skills.list should expose structured output")?;
assert!(serde_json::to_vec(&response)?.len() <= byte_budget);
let next_cursor = response["next_cursor"].as_str().map(str::to_owned);
assert_eq!(
next_cursor
.as_deref()
.and_then(|value| value.split(':').nth(1)),
cursor_offset
);
let expected_skills = expected_indices
.into_iter()
.map(|index| &complete_skills[index])
.collect::<Vec<_>>();
assert_eq!(
response,
serde_json::json!({
"skills": expected_skills,
"warnings": expected_warnings,
"next_cursor": next_cursor,
})
);
if index == 2 {
let (legacy_cursor, _) = next_cursor
.as_deref()
.and_then(|cursor| cursor.rsplit_once(':'))
.ok_or("cursor should include its budget")?;
let legacy_payload = ToolPayload::Function {
arguments: serde_json::json!({
"authority": {"kind": "cloud"},
"cursor": legacy_cursor,
})
.to_string(),
};
let output = list_tool
.handle(ToolCall {
payload: legacy_payload.clone(),
..call.clone()
})
.await?;
assert_eq!(
output.post_tool_use_response(&call.call_id, &legacy_payload),
Some(serde_json::json!({
"skills": &complete_skills[2..],
"warnings": [],
"next_cursor": null,
}))
);
}
cursor = next_cursor;
}
assert_eq!(
list_tool
.handle(ToolCall {
call_id: "omitted-call".to_string(),
truncation_policy: TruncationPolicy::Bytes(64),
..call
})
.await
.err(),
Some(FunctionCallError::RespondToModel(
"skills.list response budget leaves no room for discovery warnings".to_string()
))
);
let read_tool = tools
.iter()
.find(|tool| tool.tool_name().name == "read")
.ok_or("skills.read tool should be registered")?;
// The resource fits the 2400-byte budget before escaping, but not after.
let insufficient_budget_payload = ToolPayload::Function {
arguments: serde_json::json!({
"package": format!("cloud/{opaque_suffix}"),
})
.to_string(),
};
let error = read_tool
.handle(ToolCall {
turn_id: "turn-1".to_string(),
call_id: "insufficient-read-budget".to_string(),
tool_name: read_tool.tool_name(),
model: "gpt-test".to_string(),
codex_turn_metadata: None,
truncation_policy: TruncationPolicy::Bytes(2_000),
source: ToolCallSource::Direct,
conversation_history: ConversationHistory::default(),
turn_item_emitter: Arc::new(NoopTurnItemEmitter),
attachment_store: Arc::new(InlineAttachmentStore),
ephemeral: false,
environments: Vec::new(),
payload: insufficient_budget_payload,
})
.await
.err()
.expect("skills.read should reject a response that cannot fit any contents");
let error = assert_matches!(error, FunctionCallError::RespondToModel(error) => error);
assert_eq!(
error,
"skills.read response budget leaves no room for contents"
);
Ok(())
}
#[tokio::test]
async fn cloud_discovery_is_gated_and_reports_failures() -> TestResult {
for (cloud_skill_enabled, has_cloud_provider) in [(false, true), (true, true), (true, false)] {
let list_calls = Arc::new(AtomicUsize::new(0));
let mut providers = SkillProviders::new();
if has_cloud_provider {
providers = providers.with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog::default(),
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: Some(Arc::clone(&list_calls)),
fail_first_list: true,
}));
}
let cloud_available = cloud_skill_enabled && has_cloud_provider;
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
let config = TestConfig {
cloud_skill_enabled,
..default_config()
};
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
assert_eq!(
registry
.turn_lifecycle_contributors()
.iter()
.map(|contributor| contributor.turn_start_phase(&thread_store))
.collect::<Vec<_>>(),
vec![
codex_extension_api::TurnStartPhase::BeforeTaskRegistration,
if cloud_available {
codex_extension_api::TurnStartPhase::RegularTaskStart
} else {
codex_extension_api::TurnStartPhase::BeforeTaskRegistration
},
]
);
assert_eq!(
registry
.turn_lifecycle_contributors()
.iter()
.map(|contributor| contributor.requires_mcp_runtime(&thread_store))
.collect::<Vec<_>>(),
vec![false, cloud_available]
);
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
assert_eq!(
list_calls.load(Ordering::Relaxed),
usize::from(cloud_available)
);
for discovery_v2 in [None, Some([].as_slice())] {
let step_store = ExtensionData::new("step");
if let Some(snapshots) = discovery_v2 {
step_store.insert(Vec::<
codex_exec_server::ExecutorCapabilityDiscoveryV2Snapshot,
>::from(snapshots));
}
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &ExtensionData::new("turn-1"),
},
)
.await;
if has_cloud_provider {
// Resume may retain a catalog even when the new Cloud discovery fails.
let previous = serde_json::json!({
"body": "Previously advertised Cloud skills",
"includeInstructions": true,
"enabled": true,
});
let update = world_state_section(&sections, "cloud_skills")
.render_diff(PreviousWorldStateSection::Known(&previous))
.1
.ok_or("unavailable Cloud skills should withdraw the previous catalog")?;
assert_eq!(
update.body().clone().expect_text(),
"\n## Cloud skills update\nNo cloud skills are currently available.\n"
);
} else {
assert!(
sections
.iter()
.all(|section| section.id() != "cloud_skills")
);
}
}
if !cloud_available {
assert!(event_rx.try_recv().is_err());
let tools = registry.tool_contributors()[0].tools_for_step(
&session_store,
&thread_store,
&ExtensionData::new("step"),
);
assert!(tools.iter().all(|tool| tool.tool_name().name != "list"));
continue;
}
let warning = event_rx.try_recv()?.into_warning();
assert_eq!(warning.thread_id, thread_store.level_id());
assert_eq!(warning.turn_id.as_deref(), Some("turn-1"));
assert!(warning.message.contains("temporary cloud failure"));
}
Ok(())
}
#[tokio::test]
async fn v2_root_aliases_and_read_cache_use_the_captured_environment() -> TestResult {
let directory = tempfile::tempdir()?;
let root = directory.path().join("skills");
let path = root.join("nested/demo/SKILL.md");
std::fs::create_dir_all(path.parent().ok_or("skill directory")?)?;
std::fs::write(&path, "original\n".repeat(2_000))?;
let manager = Arc::new(EnvironmentManager::default_for_tests());
let environment = manager
.get_environment("local")
.ok_or("local environment")?;
let replacement = EnvironmentManager::default_for_tests()
.get_environment("local")
.ok_or("replacement environment")?;
let providers = SkillProviders::new().with_executor_provider(Arc::new(
ExecutorSkillProvider::new_with_restriction_product(
manager, /*restriction_product*/ None,
),
));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &default_config(),
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let snapshots = vec![ExecutorCapabilityDiscoveryV2Snapshot {
environment_id: "captured".to_string(),
result: Ok(Arc::new(DiscoverV2CapabilitiesResponse {
skills: ["demo", "build", "check", "deploy", "review", "test"]
.into_iter()
.map(|name| {
Ok(ExecutorSkill {
name: name.to_string(),
description: "Demo".to_string(),
short_description: None,
path: PathUri::from_host_native_path(
root.join(format!("nested/{name}/SKILL.md")),
)?,
discovery_root: Some(SkillDiscoveryRoot {
id: "codex-home-skills".to_string(),
path: PathUri::from_host_native_path(&root)?,
}),
dependencies: None,
policy: None,
})
})
.collect::<Result<Vec<_>, Box<dyn std::error::Error>>>()?,
..Default::default()
})),
}];
let package = format!(
"skill://codex-home-skills/{}",
PathUri::from_host_native_path(path.parent().ok_or("skill directory")?)?
.inferred_native_path_string()
.replace('\\', "/")
.trim_start_matches('/')
);
let turn_store = ExtensionData::new("turn-1");
let mut cursor = None;
for (handle, expected_error) in [
(Some(Arc::clone(&environment)), None),
(Some(environment), None),
(
Some(replacement),
Some("skills.read cursor is stale; restart from the first page"),
),
(
None,
Some("skill environment is not available for this callback"),
),
] {
let file_system = handle.as_ref().map(|handle| handle.get_filesystem());
let access = file_system
.as_ref()
.map(FileSystemEnvironmentAccessor::unrestricted);
let step_store = ExtensionData::new("turn-1");
step_store.insert(snapshots.clone());
step_store.insert(
handle
.map(|handle| ("captured".to_string(), handle))
.into_iter()
.collect::<HashMap<_, _>>(),
);
let sections = prepared_world_state(
&registry,
&[],
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
let rendered = sections
.iter()
.filter_map(|section| section.render_diff(PreviousWorldStateSection::Absent).1)
.map(|fragment| fragment.body().clone().expect_text())
.collect::<Vec<_>>()
.join("\n");
assert!(rendered.contains("e0/nested/demo"), "{rendered}");
let tools = registry.tool_contributors()[0].tools_for_step(
&session_store,
&thread_store,
&step_store,
);
let read = tools
.iter()
.find(|tool| tool.tool_name().name == "read")
.ok_or("skills.read")?;
let payload = ToolPayload::Function {
arguments: serde_json::json!({"package": "e0/nested/demo", "cursor": cursor})
.to_string(),
};
let result = read
.handle(ToolCall {
turn_id: "turn-1".to_string(),
call_id: "read".to_string(),
tool_name: read.tool_name(),
model: "gpt-test".to_string(),
codex_turn_metadata: None,
truncation_policy: TruncationPolicy::Bytes(2_000),
source: ToolCallSource::Direct,
conversation_history: ConversationHistory::default(),
turn_item_emitter: Arc::new(NoopTurnItemEmitter),
attachment_store: Arc::new(InlineAttachmentStore),
ephemeral: false,
environments: access
.iter()
.map(|access| {
codex_extension_api::ToolEnvironment::new(
"captured".to_string(),
PathUri::from_host_native_path(directory.path()).expect("fixture cwd"),
access,
)
})
.collect(),
payload: payload.clone(),
})
.await;
if let Some(expected) = expected_error {
assert!(
matches!(result, Err(FunctionCallError::RespondToModel(message)) if message == expected)
);
} else {
let response = result?
.post_tool_use_response("read", &payload)
.ok_or("read output")?;
assert_eq!(response["resource"], format!("{package}/SKILL.md"));
assert_eq!(
response["installations"],
serde_json::json!([{
"environment_id": "captured",
"skill_root": PathUri::from_host_native_path(path.parent().ok_or("skill directory")?)?
.inferred_native_path_string(),
}])
);
assert!(
response["contents"]
.as_str()
.ok_or("contents")?
.contains("original")
);
cursor = Some(
response["next_cursor"]
.as_str()
.ok_or("next cursor")?
.to_string(),
);
// The next page keeps the cached contents only while the captured handle is unchanged.
std::fs::write(&path, "updated\n".repeat(2_000))?;
}
}
Ok(())
}
#[tokio::test]
async fn root_qualified_locator_selects_only_the_matching_executor_skill() -> TestResult {
let step_store = ExtensionData::new("step-1");
let read_requests = Arc::new(Mutex::new(Vec::new()));
let root_a_locator = "skill://root-a/shared/lint-fix/SKILL.md";
let root_b_locator = "skill://root-b/shared/lint-fix/SKILL.md";
let skill_path = PathUri::parse("file:///shared/lint-fix/SKILL.md")?;
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: [("root-a", root_a_locator), ("root-b", root_b_locator)]
.into_iter()
.map(|(root_id, locator)| {
SkillCatalogEntry::new(
SkillPackageId(locator.to_string()),
SkillAuthority::new(SkillSourceKind::Executor, root_id),
"lint-fix",
"Fix lint errors.",
SkillResourceId::environment(locator, "env-1", skill_path.clone()),
)
.with_display_path(locator)
})
.collect(),
warnings: Vec::new(),
},
read_requests: Arc::clone(&read_requests),
list_calls: None,
fail_first_list: false,
});
let providers = SkillProviders::new().with_executor_provider(executor_provider);
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let selected_roots = [("root-a", "/skills/root-a"), ("root-b", "/skills/root-b")]
.into_iter()
.map(|(id, path)| SelectedCapabilityRoot {
id: id.to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse(&format!("file://{path}")).expect("skill root URI"),
},
})
.collect::<Vec<_>>();
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let turn_store = ExtensionData::new("turn-1");
prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[TurnEnvironmentSelection {
selected_capability_roots: Default::default(),
environment_id: "env-1".to_string(),
cwd: PathUri::parse("file:///workspace").expect("cwd URI"),
workspace_roots: Vec::new(),
config: EnvironmentConfigState::FromThread,
}],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &step_store,
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Mention {
name: "lint-fix".to_string(),
path: root_b_locator.to_string(),
}],
environments: vec![codex_extension_api::TurnInputEnvironment {
environment_id: "env-1".to_string(),
cwd: PathUri::parse("file:///workspace")?,
is_primary: true,
fs: &FileSystemEnvironmentAccessor::unrestricted(&LOCAL_FS),
}],
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
assert_eq!(1, fragments.len());
assert!(fragments[0].render().expect_text().contains(root_b_locator));
assert_eq!(
vec![(
SkillAuthority::new(SkillSourceKind::Executor, "root-b"),
SkillPackageId(root_b_locator.to_string()),
SkillResourceId::environment(root_b_locator, "env-1", skill_path),
)],
read_request_keys(&read_requests)
);
Ok(())
}
#[tokio::test]
async fn model_context_window_scales_executor_and_cloud_catalogs() -> TestResult {
let cloud_entries = (0..40)
.map(|index| {
test_entry(
SkillSourceKind::Cloud,
"cloud",
&format!("cloud/skill-{index:02}"),
&format!("skill-{index:02}/SKILL.md"),
)
})
.collect();
let executor_entries = (0..200)
.map(|index| {
test_entry(
SkillSourceKind::Executor,
"env-1",
&format!("executor/skill-{index:02}"),
&format!("skill-{index:02}/SKILL.md"),
)
})
.collect();
let providers = SkillProviders::new()
.with_cloud_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: cloud_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}))
.with_executor_provider(Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: executor_entries,
warnings: Vec::new(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
}));
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.bundled_skills_enabled = false;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
start_registered_turn(&registry, &session_store, &thread_store, "turn-1").await;
let mut model_info = model_info_from_slug("test-model");
model_info.context_window = Some(10_000);
let thread_fragments = registry.context_contributors()[0]
.contribute_thread_context(&session_store, &thread_store)
.await;
assert!(thread_fragments.is_empty());
let selected_roots = vec![SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills").expect("skill root URI"),
},
}];
let turn_store = ExtensionData::new("turn-1");
let sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &model_info,
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
// Core rebuilds world state before each sampling step.
let _repeated_sections = prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &ExtensionData::new("step"),
model_info: &model_info,
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
assert!(event_rx.try_recv().is_err());
let executor_section = world_state_section(&sections, "skills");
let (snapshot, executor_fragment) =
executor_section.render_diff(PreviousWorldStateSection::Absent);
let snapshot = snapshot.unwrap();
let executor_fragment = executor_fragment.ok_or("bounded executor catalog should render")?;
assert!(
!executor_fragment
.body()
.clone()
.expect_text()
.contains("skill-39")
);
let cloud_fragment = world_state_section(&sections, "cloud_skills")
.render_diff(PreviousWorldStateSection::Absent)
.1
.ok_or("bounded cloud catalog should render")?;
assert!(
cloud_fragment
.body()
.clone()
.expect_text()
.contains("additional skills omitted")
);
let warnings = event_rx
.try_iter()
.map(CapturedExtensionEvent::into_warning)
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 2);
for warning in warnings {
assert_eq!(warning.thread_id, thread_store.level_id());
assert_eq!(warning.turn_id.as_deref(), Some("turn-1"));
assert!(
warning
.message
.starts_with("Exceeded skills context budget.")
);
assert!(
warning
.message
.ends_with("additional skills were not included in the model-visible skills list.")
);
}
assert!(
executor_section
.render_diff(PreviousWorldStateSection::Known(&snapshot))
.1
.is_none()
);
assert!(event_rx.try_recv().is_err());
Ok(())
}
#[tokio::test]
async fn executor_catalog_emits_at_most_four_warnings() -> TestResult {
let step_store = ExtensionData::new("step-1");
let executor_provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: Vec::new(),
warnings: (0..6).map(|index| format!("warning-{index}")).collect(),
},
read_requests: Arc::new(Mutex::new(Vec::new())),
list_calls: None,
fail_first_list: false,
});
let providers = SkillProviders::new().with_executor_provider(executor_provider);
let (event_tx, event_rx) = std::sync::mpsc::channel();
let mut builder =
ExtensionRegistryBuilder::with_event_sink(Arc::new(ChannelEventSink(event_tx)));
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.bundled_skills_enabled = false;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let selected_roots = vec![SelectedCapabilityRoot {
id: "skills".to_string(),
location: CapabilityRootLocation::Environment {
environment_id: "env-1".to_string(),
path: PathUri::parse("file:///skills").expect("skill root URI"),
},
}];
let turn_store = ExtensionData::new("turn-1");
prepared_world_state(
&registry,
&selected_roots,
WorldStateContributionInput {
previous_world_state: None,
step_store: &step_store,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "turn-1",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
},
)
.await;
prepared_turn_input(
&registry,
TurnInputContext {
step_store: &step_store,
turn_id: "turn-1".to_string(),
user_input: Vec::new(),
environments: Vec::new(),
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
let messages = event_rx
.try_iter()
.map(|event| event.into_warning().message)
.collect::<Vec<_>>();
assert_eq!(
messages,
vec!["warning-0", "warning-1", "warning-2", "warning-3"]
);
Ok(())
}
#[tokio::test]
async fn host_catalog_compacts_shared_paths_under_budget_pressure() -> TestResult {
let test_root = test_codex_home();
let root = test_root.join(
"plugins/cache/openai-curated/example/hash1234567890/skills-with-a-very-long-shared-prefix",
);
for index in 0..12 {
let skill_dir = root.join(format!("skill-{index:02}"));
std::fs::create_dir_all(&skill_dir)?;
std::fs::write(
skill_dir.join("SKILL.md"),
format!(
"---\nname: skill-{index:02}\ndescription: Fix lint errors.\n---\n# Skill {index:02}\n"
),
)?;
}
#[cfg(unix)]
{
let source_skill_dir = test_root.join("shared-skills/linked-skill");
std::fs::create_dir_all(&source_skill_dir)?;
std::fs::write(
source_skill_dir.join("SKILL.md"),
"---\nname: linked-skill\ndescription: Linked skill.\n---\n# Linked skill\n",
)?;
std::os::unix::fs::symlink(&source_skill_dir, root.join("linked-skill"))?;
}
let root = AbsolutePathBuf::try_from(std::fs::canonicalize(root)?)?;
let rendered_root = root.to_string_lossy().replace('\\', "/");
let config_layer_stack = ConfigLayerStack::new(
vec![ConfigLayerEntry::new(
ConfigLayerSource::SessionFlags,
toml::from_str("[skills.bundled]\nenabled = false\n")?,
)],
Default::default(),
ConfigRequirementsToml::default(),
)?;
let codex_home = AbsolutePathBuf::try_from(test_root.clone())?;
let service = HostSkillsService::new_with_restriction_product(
codex_home.clone(),
/*bundled_skills_enabled*/ false,
/*restriction_product*/ None,
);
service.set_extra_roots(vec![root]);
let snapshot = service
.snapshot_for_config(
&HostSkillsLoadInput::new(codex_home, Vec::new(), config_layer_stack),
Some(Arc::clone(&LOCAL_FS)),
)
.await;
assert_eq!(snapshot.outcome().errors, Vec::new());
assert_eq!(
snapshot.outcome().skills.len(),
12 + usize::from(cfg!(unix))
);
let mut builder = ExtensionRegistryBuilder::new();
install(&mut builder, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let mut config = default_config();
config.bundled_skills_enabled = false;
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let mut model_info = model_info_from_slug("test-model");
model_info.context_window = Some(10_000);
thread_store.insert(model_info);
let turn_store = ExtensionData::new("turn-1");
turn_store.insert(snapshot);
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &ExtensionData::new("step"),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "hello".to_string(),
text_elements: Vec::new(),
}],
environments: Vec::new(),
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&turn_store,
)
.await;
let catalog = fragments
.iter()
.find(|fragment| fragment.role() == "developer")
.ok_or("host catalog should render")?
.render()
.expect_text();
assert!(
catalog.contains(&format!("- `r0` = `{rendered_root}`")),
"{catalog}"
);
assert!(catalog.contains("(file: r0/skill-00/SKILL.md)"));
assert!(catalog.contains("(file: r0/skill-11/SKILL.md)"));
#[cfg(unix)]
assert!(catalog.contains("(file: r0/linked-skill/SKILL.md)"));
assert!(!catalog.contains("additional skills omitted"));
std::fs::remove_dir_all(test_root)?;
Ok(())
}
#[tokio::test]
async fn prompt_hidden_skill_can_still_be_invoked() -> TestResult {
let read_requests = Arc::new(Mutex::new(Vec::new()));
let provider = Arc::new(StaticSkillProvider {
catalog: SkillCatalog {
is_partial: false,
entries: vec![
test_entry(
SkillSourceKind::Host,
"host",
"host/visible-skill",
"visible-skill/SKILL.md",
),
test_entry(
SkillSourceKind::Host,
"host",
"host/hidden-skill",
"hidden-skill/SKILL.md",
)
.hidden_from_prompt(),
],
warnings: Vec::new(),
},
read_requests: Arc::clone(&read_requests),
list_calls: None,
fail_first_list: false,
});
let providers = SkillProviders::new().with_host_provider(provider);
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(&mut builder, providers, skills_extension_config);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let session_source = SessionSource::Cli;
let config = default_config();
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &config,
session_source: &session_source,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
let fragments = prepared_turn_input(
&registry,
TurnInputContext {
step_store: &ExtensionData::new("step"),
turn_id: "turn-1".to_string(),
user_input: vec![UserInput::Text {
text: "$hidden-skill".to_string(),
text_elements: Vec::new(),
}],
environments: Vec::new(),
},
/*extension_metrics*/ None,
&session_store,
&thread_store,
&ExtensionData::new("turn-1"),
)
.await;
assert_eq!(2, fragments.len());
let visible = fragments[0].render().expect_text();
let hidden = fragments[1].render().expect_text();
assert!(visible.contains("visible-skill"));
assert!(!visible.contains("hidden-skill"));
assert!(hidden.contains("<name>hidden-skill</name>"));
assert_eq!(
vec![(
SkillAuthority::new(SkillSourceKind::Host, "host"),
SkillPackageId("host/hidden-skill".to_string()),
SkillResourceId::new("hidden-skill/SKILL.md"),
)],
read_request_keys(&read_requests)
);
Ok(())
}
#[derive(Clone)]
struct StaticSkillProvider {
catalog: SkillCatalog,
read_requests: Arc<Mutex<Vec<(SkillAuthority, SkillPackageId, SkillResourceId)>>>,
list_calls: Option<Arc<AtomicUsize>>,
fail_first_list: bool,
}
#[derive(Debug)]
enum CapturedExtensionEvent {
Event(Box<Event>),
Warning(ExtensionWarning),
}
impl CapturedExtensionEvent {
fn into_warning(self) -> ExtensionWarning {
match self {
Self::Warning(warning) => warning,
Self::Event(event) => panic!("expected extension warning, got {event:?}"),
}
}
}
struct ChannelEventSink(std::sync::mpsc::Sender<CapturedExtensionEvent>);
impl ExtensionEventSink for ChannelEventSink {
fn emit(&self, event: Event) {
let _ = self.0.send(CapturedExtensionEvent::Event(Box::new(event)));
}
fn emit_warning(&self, warning: ExtensionWarning) {
let _ = self.0.send(CapturedExtensionEvent::Warning(warning));
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct RecordedHistogram {
name: String,
value: i64,
tags: Vec<(String, String)>,
}
#[derive(Default)]
struct RecordingMetrics {
samples: Mutex<Vec<RecordedHistogram>>,
}
impl RecordingMetrics {
fn samples(&self) -> Vec<RecordedHistogram> {
self.samples
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
}
impl ExtensionMetrics for RecordingMetrics {
fn histogram_with_boundaries(
&self,
name: &str,
value: i64,
_boundaries: &[f64],
tags: &[(&str, &str)],
) {
self.histogram(name, value, tags);
}
fn counter(&self, name: &str, _inc: i64, _tags: &[(&str, &str)]) {
panic!("unexpected counter: {name}");
}
fn histogram(&self, name: &str, value: i64, tags: &[(&str, &str)]) {
self.samples
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(RecordedHistogram {
name: name.to_string(),
value,
tags: tags
.iter()
.map(|(key, value)| ((*key).to_string(), (*value).to_string()))
.collect(),
});
}
}
fn expected_catalog_entries(stage: &str, status: &str, count: i64) -> RecordedHistogram {
RecordedHistogram {
name: "codex.skills.catalog.entries".to_string(),
value: count,
tags: vec![
("stage".to_string(), stage.to_string()),
("status".to_string(), status.to_string()),
],
}
}
fn expected_catalog_metric_samples(catalog_surface: &str, count: i64) -> Vec<RecordedHistogram> {
let tags = vec![("catalog_surface".to_string(), catalog_surface.to_string())];
vec![
RecordedHistogram {
name: THREAD_SKILLS_ENABLED_TOTAL_METRIC.to_string(),
value: count,
tags: tags.clone(),
},
RecordedHistogram {
name: THREAD_SKILLS_KEPT_TOTAL_METRIC.to_string(),
value: count,
tags: tags.clone(),
},
RecordedHistogram {
name: THREAD_SKILLS_TRUNCATED_METRIC.to_string(),
value: 0,
tags: tags.clone(),
},
RecordedHistogram {
name: THREAD_SKILLS_DESCRIPTION_TRUNCATED_CHARS_METRIC.to_string(),
value: 0,
tags,
},
]
}
impl SkillProvider for StaticSkillProvider {
fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> {
let list_call = self
.list_calls
.as_ref()
.map(|list_calls| list_calls.fetch_add(1, Ordering::Relaxed));
let fail = self.fail_first_list && list_call == Some(0);
let catalog = self.catalog.clone();
Box::pin(async move {
if fail {
Err(SkillProviderError::new("temporary cloud failure"))
} else {
Ok(catalog)
}
})
}
fn read<'a>(
&'a self,
request: SkillReadRequest<'a>,
) -> SkillProviderFuture<'a, SkillReadResult> {
let read_requests = Arc::clone(&self.read_requests);
Box::pin(async move {
read_requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push((
request.authority.clone(),
request.package.clone(),
request.resource.clone(),
));
Ok(SkillReadResult {
resource: request.resource,
contents: "# Lint Fix\n\nRun the formatter.".to_string(),
})
})
}
fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> {
Box::pin(async { Ok(SkillSearchResult::default()) })
}
}
fn test_entry(
kind: SkillSourceKind,
authority_id: &str,
package_id: &str,
main_prompt: &str,
) -> SkillCatalogEntry {
let name = package_id.rsplit('/').next().unwrap_or(package_id);
SkillCatalogEntry::new(
SkillPackageId(package_id.to_string()),
SkillAuthority::new(kind, authority_id),
name,
"Fix lint errors.",
SkillResourceId::new(main_prompt),
)
.with_display_path(format!("skill://{package_id}/SKILL.md"))
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct TestConfig {
include_instructions: bool,
catalog_deltas_enabled: bool,
omit_descriptions: bool,
bundled_skills_enabled: bool,
cloud_skill_enabled: bool,
shadow_selection_enabled: bool,
}
fn default_config() -> TestConfig {
TestConfig {
include_instructions: true,
catalog_deltas_enabled: false,
omit_descriptions: false,
bundled_skills_enabled: true,
cloud_skill_enabled: true,
shadow_selection_enabled: false,
}
}
fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig {
SkillsExtensionConfig {
include_instructions: config.include_instructions,
catalog_deltas_enabled: config.catalog_deltas_enabled,
omit_descriptions: config.omit_descriptions,
max_context_tokens: None,
bundled_skills_enabled: config.bundled_skills_enabled,
cloud_skill_enabled: config.cloud_skill_enabled,
shadow_selection_enabled: config.shadow_selection_enabled,
}
}
fn test_codex_home() -> PathBuf {
let id = NEXT_CODEX_HOME_ID.fetch_add(1, Ordering::Relaxed);
std::env::temp_dir().join(format!(
"codex-skills-extension-test-{}-{id}",
std::process::id(),
))
}
fn read_request_keys(
requests: &Mutex<Vec<(SkillAuthority, SkillPackageId, SkillResourceId)>>,
) -> Vec<(SkillAuthority, SkillPackageId, SkillResourceId)> {
requests
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
#[path = "skills_extension/merge_catalog_tests.rs"]
mod merge_catalog_tests;
#[path = "skills_extension/discovery_v2_tests.rs"]
mod discovery_v2_tests;
#[tokio::test]
async fn shared_catalog_keeps_captured_executor_and_separates_bindings_in_one_turn() -> TestResult {
let root_dir = tempfile::tempdir()?;
let skill_dir = root_dir.path().join("demo");
std::fs::create_dir_all(&skill_dir)?;
std::fs::write(skill_dir.join("SKILL.md"), DEMO_SKILL_CONTENTS)?;
let manager = Arc::new(EnvironmentManager::default_for_tests());
let original = manager.get_environment("local").ok_or("local executor")?;
let replacement = EnvironmentManager::default_for_tests()
.get_environment("local")
.ok_or("replacement")?;
let roots = vec![SelectedCapabilityRoot {
id: "skills".into(),
location: CapabilityRootLocation::Environment {
environment_id: "captured".into(),
path: PathUri::from_host_native_path(root_dir.path())?,
},
}];
let captured = manager
.resolve_selected_capability_roots(
&roots,
&HashMap::from([("captured".into(), Some(original))]),
)
.await;
let newer = manager
.resolve_selected_capability_roots(
&roots,
&HashMap::from([("captured".into(), Some(replacement))]),
)
.await;
let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
let address = listener.local_addr()?;
drop(listener);
manager.upsert_environment(
"captured".into(),
format!("ws://{address}"),
Some(std::time::Duration::from_millis(100)),
)?;
let provider = Arc::new(ExecutorSkillProvider::new_with_restriction_product(
manager, /*restriction_product*/ None,
));
let mut builder = ExtensionRegistryBuilder::new();
install_with_providers(
&mut builder,
SkillProviders::new().with_executor_provider(provider),
skills_extension_config,
);
let registry = builder.build();
let session_store = ExtensionData::new("session");
let thread_store = ExtensionData::new("thread");
let turn_store = ExtensionData::new("same-turn");
registry.thread_lifecycle_contributors()[0]
.on_thread_start(ThreadStartInput {
config: &default_config(),
session_source: &SessionSource::Cli,
persistent_thread_state_available: true,
environments: &[],
mcp_resource_client: None,
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
})
.await;
for (index, (resolved, expected)) in [
(captured.clone(), Some("Demo skill.")),
(captured, Some("Demo skill.")),
(newer, Some("New executor inventory.")),
(Vec::new(), None),
]
.into_iter()
.enumerate()
{
if index == 1 {
std::fs::write(
skill_dir.join("SKILL.md"),
"---\nname: demo\ndescription: New executor inventory.\n---\nNew body\n",
)?;
}
let step_store = ExtensionData::new(format!("step-{index}"));
step_store.insert(resolved);
let sections = prepared_world_state(
&registry,
&roots,
WorldStateContributionInput {
previous_world_state: None,
model_info: &catalog_model_info(),
thread_id: codex_protocol::ThreadId::new(),
turn_id: "same-turn",
environments: &[],
extension_metrics: None,
session_store: &session_store,
thread_store: &thread_store,
turn_store: &turn_store,
step_store: &step_store,
},
)
.await;
let rendered = sections
.iter()
.filter_map(|section| section.render_diff(PreviousWorldStateSection::Absent).1)
.map(|fragment| fragment.body().clone().expect_text())
.collect::<Vec<_>>()
.join("\n");
if let Some(expected) = expected {
assert!(rendered.contains(expected), "step {index}: {rendered}");
} else {
assert!(!rendered.contains("- demo:"), "{rendered}");
}
}
Ok(())
}
#[path = "skills_extension/preparation_tests.rs"]
mod preparation_tests;
Reference in New Issue View Git Blame Copy Permalink