mirror of
https://github.com/openai/codex.git
synced 2026-11-21 22:50:37 +00:00
Add shadow metrics for lexical skill selection (#32761)
## What changed - Add an opt-in `skill_search` feature that ranks prompt-visible skills against each turn's user input with a bounded weighted lexical selector. - Keep the ranked selection out of model-visible context and record metrics for selection cost, catalog reduction, and whether later implicit or `skills.read` invocations matched the ranked candidates. - Include host-provided skills in the experiment catalog without changing the rendered skill catalog. ## Testing - Add selector unit tests covering ranking, limits, truncation, stop words, and deterministic tie-breaking. - Add extension tests covering turn-local invocation recording and host-skill shadow selection. GitOrigin-RevId: 4d00a1c805ea8b391d6c6ac6a8450afa88ca3e25
This commit is contained in:
Generated
+2
@@ -3979,11 +3979,13 @@ dependencies = [
|
||||
"codex-exec-server",
|
||||
"codex-extension-api",
|
||||
"codex-mcp",
|
||||
"codex-otel",
|
||||
"codex-protocol",
|
||||
"codex-tools",
|
||||
"codex-utils-absolute-path",
|
||||
"codex-utils-path-uri",
|
||||
"codex-utils-string",
|
||||
"opentelemetry_sdk",
|
||||
"pretty_assertions",
|
||||
"schemars 0.8.22",
|
||||
"serde",
|
||||
|
||||
@@ -84,13 +84,17 @@ where
|
||||
.with_orchestrator_provider(Arc::new(
|
||||
codex_skills_extension::OrchestratorSkillProvider::new(),
|
||||
));
|
||||
codex_skills_extension::install_with_providers(
|
||||
codex_skills_extension::install_with_providers_and_metrics(
|
||||
&mut builder,
|
||||
skill_providers,
|
||||
codex_otel::global(),
|
||||
|config: &Config| codex_skills_extension::SkillsExtensionConfig {
|
||||
include_instructions: config.include_skill_instructions,
|
||||
bundled_skills_enabled: config.bundled_skills_enabled(),
|
||||
orchestrator_skills_enabled: config.orchestrator_skills_enabled,
|
||||
shadow_selection_enabled: config
|
||||
.features
|
||||
.enabled(codex_features::Feature::SkillSearch),
|
||||
},
|
||||
);
|
||||
Arc::new(builder.build())
|
||||
|
||||
@@ -650,6 +650,9 @@
|
||||
"skill_mcp_dependency_install": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"skill_search": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"sqlite": {
|
||||
"type": "boolean"
|
||||
},
|
||||
@@ -5026,6 +5029,9 @@
|
||||
"skill_mcp_dependency_install": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"skill_search": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"sqlite": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -7,7 +7,6 @@ version.workspace = true
|
||||
[lib]
|
||||
name = "codex_skills_extension"
|
||||
path = "src/lib.rs"
|
||||
test = false
|
||||
doctest = false
|
||||
|
||||
[lints]
|
||||
@@ -18,6 +17,7 @@ codex-core-skills = { workspace = true }
|
||||
codex-exec-server = { workspace = true }
|
||||
codex-extension-api = { workspace = true }
|
||||
codex-mcp = { workspace = true }
|
||||
codex-otel = { workspace = true }
|
||||
codex-protocol = { workspace = true }
|
||||
codex-tools = { workspace = true }
|
||||
codex-utils-path-uri = { workspace = true }
|
||||
@@ -31,5 +31,6 @@ url = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
codex-utils-absolute-path = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true }
|
||||
pretty_assertions = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
||||
|
||||
@@ -7,4 +7,6 @@ pub struct SkillsExtensionConfig {
|
||||
pub bundled_skills_enabled: bool,
|
||||
/// Whether orchestrator-owned skills are eligible for discovery.
|
||||
pub orchestrator_skills_enabled: bool,
|
||||
/// Whether cheap skill selectors run in shadow mode without changing prompt contents.
|
||||
pub shadow_selection_enabled: bool,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
mod weighted_lexical;
|
||||
pub(crate) use weighted_lexical::WeightedLexicalSkillSelector;
|
||||
|
||||
/// Metadata searched by a cheap skill selector.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct SkillSelectionDocument<'a> {
|
||||
/// Caller-owned identifier returned in [`CheapSkillSelection::candidate_ids`].
|
||||
pub id: usize,
|
||||
pub name: &'a str,
|
||||
pub short_description: Option<&'a str>,
|
||||
pub description: &'a str,
|
||||
}
|
||||
|
||||
/// Bounded output from one cheap skill-selection method.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct CheapSkillSelection {
|
||||
pub candidate_ids: Vec<usize>,
|
||||
pub query_term_count: usize,
|
||||
pub query_truncated: bool,
|
||||
pub candidate_set_truncated: bool,
|
||||
}
|
||||
|
||||
/// Selects likely-relevant skills without changing the model-visible catalog.
|
||||
///
|
||||
/// Implementations must be deterministic, side-effect free, and cheap enough to run in shadow
|
||||
/// mode on every turn. Callers must validate returned IDs against the supplied documents.
|
||||
pub(crate) trait CheapSkillSelector: Send + Sync {
|
||||
/// Low-cardinality identifier suitable for experiment metrics.
|
||||
fn method(&self) -> &'static str;
|
||||
|
||||
fn select(
|
||||
&self,
|
||||
query: &str,
|
||||
documents: &[SkillSelectionDocument<'_>],
|
||||
limit: usize,
|
||||
) -> CheapSkillSelection;
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::CheapSkillSelection;
|
||||
use super::CheapSkillSelector;
|
||||
use super::SkillSelectionDocument;
|
||||
|
||||
const MAX_QUERY_BYTES: usize = 4 * 1024;
|
||||
const MAX_QUERY_TERMS: usize = 64;
|
||||
const MAX_DOCUMENT_BYTES: usize = 4 * 1024;
|
||||
const MAX_DOCUMENT_TERMS: usize = 256;
|
||||
const MAX_CANDIDATES: usize = 1_000;
|
||||
const MAX_RESULTS: usize = 50;
|
||||
|
||||
const STOP_WORDS: &[&str] = &[
|
||||
"a", "an", "and", "are", "as", "at", "be", "by", "do", "for", "from", "how", "i", "in", "is",
|
||||
"it", "me", "my", "of", "on", "or", "please", "that", "the", "this", "to", "use", "we", "what",
|
||||
"when", "where", "which", "with", "you", "your",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub(crate) struct WeightedLexicalSkillSelector;
|
||||
|
||||
impl CheapSkillSelector for WeightedLexicalSkillSelector {
|
||||
fn method(&self) -> &'static str {
|
||||
"weighted_lexical_v1"
|
||||
}
|
||||
|
||||
fn select(
|
||||
&self,
|
||||
query: &str,
|
||||
documents: &[SkillSelectionDocument<'_>],
|
||||
limit: usize,
|
||||
) -> CheapSkillSelection {
|
||||
let (query, query_bytes_truncated) = bounded(query, MAX_QUERY_BYTES);
|
||||
let query_phrase = normalize_phrase(query);
|
||||
let (query_terms, query_terms_truncated) = query_terms(&query_phrase);
|
||||
let query_truncated = query_bytes_truncated || query_terms_truncated;
|
||||
let candidate_set_truncated = documents.len() > MAX_CANDIDATES;
|
||||
if query_terms.is_empty() || limit == 0 {
|
||||
return CheapSkillSelection {
|
||||
query_term_count: query_terms.len(),
|
||||
query_truncated,
|
||||
candidate_set_truncated,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
|
||||
let mut scored = documents
|
||||
.iter()
|
||||
.take(MAX_CANDIDATES)
|
||||
.filter_map(|document| {
|
||||
let score = score_document(&query_phrase, &query_terms, document);
|
||||
(score > 0).then_some((score, document.id, document.name))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
scored.sort_by(|left, right| {
|
||||
right
|
||||
.0
|
||||
.cmp(&left.0)
|
||||
.then_with(|| left.2.cmp(right.2))
|
||||
.then_with(|| left.1.cmp(&right.1))
|
||||
});
|
||||
|
||||
CheapSkillSelection {
|
||||
candidate_ids: scored
|
||||
.into_iter()
|
||||
.take(limit.min(MAX_RESULTS))
|
||||
.map(|(_, id, _)| id)
|
||||
.collect(),
|
||||
query_term_count: query_terms.len(),
|
||||
query_truncated,
|
||||
candidate_set_truncated,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn score_document(
|
||||
query_phrase: &str,
|
||||
query_terms: &[&str],
|
||||
document: &SkillSelectionDocument<'_>,
|
||||
) -> u32 {
|
||||
let name = normalize_bounded(document.name);
|
||||
let short_description = document
|
||||
.short_description
|
||||
.map(normalize_bounded)
|
||||
.unwrap_or_default();
|
||||
let description = normalize_bounded(document.description);
|
||||
let name_terms = phrase_terms(&name);
|
||||
let short_description_terms = phrase_terms(&short_description);
|
||||
let description_terms = phrase_terms(&description);
|
||||
|
||||
let mut score = 0u32;
|
||||
if !name.is_empty() && contains_phrase(query_phrase, &name) {
|
||||
score = score.saturating_add(256);
|
||||
}
|
||||
|
||||
let mut matched_query_terms = 0u32;
|
||||
for query_term in query_terms {
|
||||
let mut matched = false;
|
||||
if name == *query_term {
|
||||
score = score.saturating_add(128);
|
||||
matched = true;
|
||||
} else if name_terms.contains(query_term) {
|
||||
score = score.saturating_add(64);
|
||||
matched = true;
|
||||
} else if contains_related_term(&name_terms, query_term) {
|
||||
score = score.saturating_add(24);
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if short_description_terms.contains(query_term) {
|
||||
score = score.saturating_add(16);
|
||||
matched = true;
|
||||
} else if contains_related_term(&short_description_terms, query_term) {
|
||||
score = score.saturating_add(6);
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if description_terms.contains(query_term) {
|
||||
score = score.saturating_add(4);
|
||||
matched = true;
|
||||
} else if contains_related_term(&description_terms, query_term) {
|
||||
score = score.saturating_add(1);
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if matched {
|
||||
matched_query_terms = matched_query_terms.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
score.saturating_add(matched_query_terms.saturating_mul(matched_query_terms))
|
||||
}
|
||||
|
||||
fn normalize_bounded(value: &str) -> String {
|
||||
normalize_phrase(bounded(value, MAX_DOCUMENT_BYTES).0)
|
||||
}
|
||||
|
||||
fn normalize_phrase(value: &str) -> String {
|
||||
let mut normalized = String::with_capacity(value.len());
|
||||
let mut previous_was_separator = true;
|
||||
for character in value.chars() {
|
||||
if character.is_alphanumeric() {
|
||||
normalized.extend(character.to_lowercase());
|
||||
previous_was_separator = false;
|
||||
} else if !previous_was_separator {
|
||||
normalized.push(' ');
|
||||
previous_was_separator = true;
|
||||
}
|
||||
}
|
||||
if previous_was_separator {
|
||||
normalized.pop();
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn query_terms(query_phrase: &str) -> (Vec<&str>, bool) {
|
||||
let mut seen = HashSet::new();
|
||||
let mut terms = Vec::new();
|
||||
for term in query_phrase
|
||||
.split_whitespace()
|
||||
.filter(|term| term.chars().count() >= 2 && !STOP_WORDS.contains(term))
|
||||
{
|
||||
if !seen.insert(term) {
|
||||
continue;
|
||||
}
|
||||
if terms.len() == MAX_QUERY_TERMS {
|
||||
return (terms, true);
|
||||
}
|
||||
terms.push(term);
|
||||
}
|
||||
(terms, false)
|
||||
}
|
||||
|
||||
fn phrase_terms(phrase: &str) -> HashSet<&str> {
|
||||
phrase.split_whitespace().take(MAX_DOCUMENT_TERMS).collect()
|
||||
}
|
||||
|
||||
fn contains_phrase(haystack: &str, needle: &str) -> bool {
|
||||
haystack == needle
|
||||
|| haystack.starts_with(&format!("{needle} "))
|
||||
|| haystack.ends_with(&format!(" {needle}"))
|
||||
|| haystack.contains(&format!(" {needle} "))
|
||||
}
|
||||
|
||||
fn contains_related_term(terms: &HashSet<&str>, query_term: &str) -> bool {
|
||||
if query_term.chars().count() < 4 {
|
||||
return false;
|
||||
}
|
||||
terms.iter().any(|term| {
|
||||
term.chars().count() >= 4 && (term.starts_with(query_term) || query_term.starts_with(*term))
|
||||
})
|
||||
}
|
||||
|
||||
fn bounded(value: &str, max_bytes: usize) -> (&str, bool) {
|
||||
if value.len() <= max_bytes {
|
||||
return (value, false);
|
||||
}
|
||||
let mut end = max_bytes;
|
||||
while !value.is_char_boundary(end) {
|
||||
end = end.saturating_sub(1);
|
||||
}
|
||||
(&value[..end], true)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "weighted_lexical_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,166 @@
|
||||
use super::*;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_prioritizes_an_exact_skill_name() {
|
||||
let documents = [
|
||||
SkillSelectionDocument {
|
||||
id: 10,
|
||||
name: "slides-helper",
|
||||
short_description: None,
|
||||
description: "Create presentations and visual decks.",
|
||||
},
|
||||
SkillSelectionDocument {
|
||||
id: 20,
|
||||
name: "presentations",
|
||||
short_description: None,
|
||||
description: "Create or edit PowerPoint presentations.",
|
||||
},
|
||||
SkillSelectionDocument {
|
||||
id: 30,
|
||||
name: "spreadsheets",
|
||||
short_description: None,
|
||||
description: "Analyze tabular data.",
|
||||
},
|
||||
];
|
||||
|
||||
let selection = WeightedLexicalSkillSelector.select(
|
||||
"Use presentations to create a deck",
|
||||
&documents,
|
||||
/*limit*/ 20,
|
||||
);
|
||||
|
||||
assert_eq!(vec![20, 10], selection.candidate_ids);
|
||||
assert!(!selection.query_truncated);
|
||||
assert!(!selection.candidate_set_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_uses_descriptions_and_drops_zero_score_candidates() {
|
||||
let documents = [
|
||||
SkillSelectionDocument {
|
||||
id: 1,
|
||||
name: "ci-helper",
|
||||
short_description: Some("Diagnose continuous integration failures."),
|
||||
description: "Inspect failing GitHub Actions checks and logs.",
|
||||
},
|
||||
SkillSelectionDocument {
|
||||
id: 2,
|
||||
name: "document-editor",
|
||||
short_description: None,
|
||||
description: "Edit Word documents.",
|
||||
},
|
||||
];
|
||||
|
||||
let selection = WeightedLexicalSkillSelector.select(
|
||||
"Please diagnose the failing GitHub Actions check",
|
||||
&documents,
|
||||
/*limit*/ 20,
|
||||
);
|
||||
|
||||
assert_eq!(vec![1], selection.candidate_ids);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_respects_requested_limit() {
|
||||
let names = (0..10)
|
||||
.map(|index| format!("lint-{index}"))
|
||||
.collect::<Vec<_>>();
|
||||
let documents = names
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(id, name)| SkillSelectionDocument {
|
||||
id,
|
||||
name,
|
||||
short_description: None,
|
||||
description: "Fix lint errors.",
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let selection =
|
||||
WeightedLexicalSkillSelector.select("fix lint errors", &documents, /*limit*/ 3);
|
||||
|
||||
assert_eq!(3, selection.candidate_ids.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_reports_bounded_inputs() {
|
||||
let long_query = "match ".repeat(MAX_QUERY_BYTES);
|
||||
let names = (0..=MAX_CANDIDATES)
|
||||
.map(|index| format!("candidate-{index}"))
|
||||
.collect::<Vec<_>>();
|
||||
let documents = names
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(id, name)| SkillSelectionDocument {
|
||||
id,
|
||||
name,
|
||||
short_description: None,
|
||||
description: "match",
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let selection = WeightedLexicalSkillSelector.select(&long_query, &documents, /*limit*/ 20);
|
||||
|
||||
assert!(selection.query_truncated);
|
||||
assert!(selection.candidate_set_truncated);
|
||||
assert_eq!(20, selection.candidate_ids.len());
|
||||
assert!(
|
||||
selection
|
||||
.candidate_ids
|
||||
.iter()
|
||||
.all(|id| *id < MAX_CANDIDATES)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_caps_query_terms() {
|
||||
let query = (0..=MAX_QUERY_TERMS)
|
||||
.map(|index| format!("term{index}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
let documents = [SkillSelectionDocument {
|
||||
id: 1,
|
||||
name: "term0",
|
||||
short_description: None,
|
||||
description: "term0",
|
||||
}];
|
||||
|
||||
let selection = WeightedLexicalSkillSelector.select(&query, &documents, /*limit*/ 20);
|
||||
|
||||
assert_eq!(MAX_QUERY_TERMS, selection.query_term_count);
|
||||
assert!(selection.query_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lexical_selector_returns_nothing_for_stop_words_only() {
|
||||
let documents = [SkillSelectionDocument {
|
||||
id: 1,
|
||||
name: "anything",
|
||||
short_description: None,
|
||||
description: "Do anything.",
|
||||
}];
|
||||
|
||||
let selection =
|
||||
WeightedLexicalSkillSelector.select("please use the", &documents, /*limit*/ 20);
|
||||
|
||||
assert_eq!(CheapSkillSelection::default(), selection);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selector_can_be_used_behind_the_shared_trait() {
|
||||
fn run(selector: &dyn CheapSkillSelector) -> CheapSkillSelection {
|
||||
selector.select(
|
||||
"review code",
|
||||
&[SkillSelectionDocument {
|
||||
id: 7,
|
||||
name: "code-review",
|
||||
short_description: None,
|
||||
description: "Review code.",
|
||||
}],
|
||||
/*limit*/ 20,
|
||||
)
|
||||
}
|
||||
|
||||
assert_eq!(vec![7], run(&WeightedLexicalSkillSelector).candidate_ids);
|
||||
}
|
||||
@@ -11,6 +11,9 @@ use codex_extension_api::ExtensionEventSink;
|
||||
use codex_extension_api::ExtensionFuture;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::PromptFragment;
|
||||
use codex_extension_api::SkillInvocationContributor;
|
||||
use codex_extension_api::SkillInvocationInput;
|
||||
use codex_extension_api::SkillInvocationKind;
|
||||
use codex_extension_api::ThreadLifecycleContributor;
|
||||
use codex_extension_api::ThreadStartInput;
|
||||
use codex_extension_api::ToolCall;
|
||||
@@ -21,6 +24,7 @@ use codex_extension_api::TurnInputContributor;
|
||||
use codex_extension_api::WorldStateContributionInput;
|
||||
use codex_extension_api::WorldStateSectionContribution;
|
||||
use codex_mcp::McpResourceClient;
|
||||
use codex_otel::MetricsClient;
|
||||
use codex_protocol::openai_models::ModelInfo;
|
||||
use codex_protocol::protocol::Event;
|
||||
use codex_protocol::protocol::EventMsg;
|
||||
@@ -34,6 +38,7 @@ use crate::catalog::SkillSourceKind;
|
||||
use crate::fragments::SkillInstructions;
|
||||
use crate::provider::HostSkillProvider;
|
||||
use crate::provider::SkillListQuery;
|
||||
use crate::provider::SkillProvider;
|
||||
use crate::provider::SkillReadRequest;
|
||||
use crate::render::MAX_SKILL_NAME_BYTES;
|
||||
use crate::render::MAX_SKILL_PATH_BYTES;
|
||||
@@ -41,6 +46,7 @@ use crate::render::available_skills_fragment;
|
||||
use crate::render::truncate_main_prompt_contents;
|
||||
use crate::render::truncate_utf8_to_bytes;
|
||||
use crate::selection::collect_explicit_skill_mentions;
|
||||
use crate::shadow_selection_experiment::ShadowSelectionExperiment;
|
||||
use crate::sources::SkillProviders;
|
||||
use crate::state::ExecutorSkillsStepState;
|
||||
use crate::state::SkillsThreadState;
|
||||
@@ -52,6 +58,7 @@ struct SkillsExtension<C> {
|
||||
providers: SkillProviders,
|
||||
event_sink: Arc<dyn ExtensionEventSink>,
|
||||
config_from_host: Arc<dyn Fn(&C) -> SkillsExtensionConfig + Send + Sync>,
|
||||
shadow_selection: Arc<ShadowSelectionExperiment>,
|
||||
}
|
||||
|
||||
impl<C> ThreadLifecycleContributor<C> for SkillsExtension<C>
|
||||
@@ -201,10 +208,37 @@ where
|
||||
self.providers.clone(),
|
||||
session_store.get::<McpResourceClient>(),
|
||||
thread_state,
|
||||
Arc::clone(&self.shadow_selection),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<C> SkillInvocationContributor for SkillsExtension<C>
|
||||
where
|
||||
C: Send + Sync + 'static,
|
||||
{
|
||||
fn on_skill_invocation<'a>(
|
||||
&'a self,
|
||||
input: SkillInvocationInput<'a>,
|
||||
) -> ExtensionFuture<'a, ()> {
|
||||
Box::pin(async move {
|
||||
match input.kind {
|
||||
SkillInvocationKind::Implicit => {
|
||||
if let Some(state) = input
|
||||
.thread_store
|
||||
.get::<SkillsThreadState>()
|
||||
.and_then(|state| state.shadow_selection_turn(input.turn_id))
|
||||
{
|
||||
self.shadow_selection
|
||||
.record_invocation(&state, input.skill_resource);
|
||||
}
|
||||
}
|
||||
SkillInvocationKind::Explicit => {}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<C> TurnInputContributor for SkillsExtension<C>
|
||||
where
|
||||
C: Send + Sync + 'static,
|
||||
@@ -232,6 +266,7 @@ where
|
||||
include_orchestrator_skills: thread_state.orchestrator_skills_enabled(),
|
||||
mcp_resources: session_store.get::<McpResourceClient>(),
|
||||
};
|
||||
let host_query = query.clone();
|
||||
let mut catalog = self.list_skills(query, &thread_state).await;
|
||||
if let Some(executor_skills) = turn_store.get::<ExecutorSkillsStepState>() {
|
||||
catalog.extend(executor_skills.0.clone());
|
||||
@@ -241,6 +276,24 @@ where
|
||||
}
|
||||
|
||||
let selected_entries = collect_explicit_skill_mentions(&input.user_input, &catalog);
|
||||
let shadow_selection_turn = if config.shadow_selection_enabled {
|
||||
// App-server leaves host discovery in core, so its configured providers omit the
|
||||
// host snapshot. Extend only the experiment catalog to keep rendered context intact.
|
||||
let mut shadow_catalog = catalog.clone();
|
||||
if host_snapshot.is_some()
|
||||
&& let Ok(host_catalog) = HostSkillProvider::new().list(host_query).await
|
||||
{
|
||||
shadow_catalog.extend(host_catalog);
|
||||
}
|
||||
Some(
|
||||
self.shadow_selection
|
||||
.run(&input.user_input, &shadow_catalog),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
thread_state
|
||||
.replace_shadow_selection_turn(input.turn_id.clone(), shadow_selection_turn);
|
||||
let mut fragments: Vec<Box<dyn ContextualUserFragment + Send>> = Vec::new();
|
||||
if config.include_instructions {
|
||||
let mut turn_catalog = catalog.clone();
|
||||
@@ -406,15 +459,33 @@ pub fn install_with_providers<C>(
|
||||
config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static,
|
||||
) where
|
||||
C: Send + Sync + 'static,
|
||||
{
|
||||
install_with_providers_and_metrics(
|
||||
registry,
|
||||
providers,
|
||||
/*metrics_client*/ None,
|
||||
config_from_host,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn install_with_providers_and_metrics<C>(
|
||||
registry: &mut ExtensionRegistryBuilder<C>,
|
||||
providers: SkillProviders,
|
||||
metrics_client: Option<MetricsClient>,
|
||||
config_from_host: impl Fn(&C) -> SkillsExtensionConfig + Send + Sync + 'static,
|
||||
) where
|
||||
C: Send + Sync + 'static,
|
||||
{
|
||||
let extension = Arc::new(SkillsExtension {
|
||||
providers,
|
||||
event_sink: registry.event_sink(),
|
||||
config_from_host: Arc::new(config_from_host),
|
||||
shadow_selection: Arc::new(ShadowSelectionExperiment::new(metrics_client)),
|
||||
});
|
||||
registry.thread_lifecycle_contributor(extension.clone());
|
||||
registry.config_contributor(extension.clone());
|
||||
registry.prompt_contributor(extension.clone());
|
||||
registry.turn_input_contributor(extension.clone());
|
||||
registry.skill_invocation_contributor(extension.clone());
|
||||
registry.tool_contributor(extension);
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
pub mod catalog;
|
||||
mod config;
|
||||
mod dynamic_skill_selector;
|
||||
mod extension;
|
||||
mod fragments;
|
||||
pub mod provider;
|
||||
mod render;
|
||||
mod selection;
|
||||
mod shadow_selection_experiment;
|
||||
mod sources;
|
||||
mod state;
|
||||
mod tools;
|
||||
@@ -13,6 +15,7 @@ mod world_state;
|
||||
pub use config::SkillsExtensionConfig;
|
||||
pub use extension::install;
|
||||
pub use extension::install_with_providers;
|
||||
pub use extension::install_with_providers_and_metrics;
|
||||
pub use provider::ExecutorSkillProvider;
|
||||
pub use provider::HostSkillProvider;
|
||||
pub use provider::OrchestratorSkillProvider;
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
// This shadow-selection experiment is temporary and should be removed after evaluation.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::PoisonError;
|
||||
use std::time::Duration;
|
||||
use std::time::Instant;
|
||||
|
||||
use codex_otel::MetricsClient;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
|
||||
use crate::catalog::SkillCatalog;
|
||||
use crate::dynamic_skill_selector::CheapSkillSelection;
|
||||
use crate::dynamic_skill_selector::CheapSkillSelector;
|
||||
use crate::dynamic_skill_selector::SkillSelectionDocument;
|
||||
use crate::dynamic_skill_selector::WeightedLexicalSkillSelector;
|
||||
|
||||
const MAX_SHADOW_QUERY_BYTES: usize = 16 * 1024;
|
||||
const MAX_SHADOW_RESULTS: usize = 20;
|
||||
|
||||
const RUN_METRIC: &str = "codex.skills.shadow_selection";
|
||||
const DURATION_METRIC: &str = "codex.skills.shadow_selection.duration_ms";
|
||||
const CATALOG_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.catalog_entries";
|
||||
const SELECTED_ENTRY_COUNT_METRIC: &str = "codex.skills.shadow_selection.selected_entries";
|
||||
const QUERY_TERM_COUNT_METRIC: &str = "codex.skills.shadow_selection.query_terms";
|
||||
const REDUCTION_BPS_METRIC: &str = "codex.skills.shadow_selection.reduction_bps";
|
||||
const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation";
|
||||
|
||||
pub(crate) struct ShadowSelectionExperiment {
|
||||
selectors: Vec<Box<dyn CheapSkillSelector>>,
|
||||
metrics_client: Option<MetricsClient>,
|
||||
}
|
||||
|
||||
impl ShadowSelectionExperiment {
|
||||
pub(crate) fn new(metrics_client: Option<MetricsClient>) -> Self {
|
||||
Self {
|
||||
selectors: vec![Box::new(WeightedLexicalSkillSelector)],
|
||||
metrics_client,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn run(
|
||||
&self,
|
||||
inputs: &[UserInput],
|
||||
catalog: &SkillCatalog,
|
||||
) -> ShadowSelectionTurnState {
|
||||
let query = build_shadow_query(inputs);
|
||||
let query_script = query_script_tag(&query.text);
|
||||
let documents = catalog
|
||||
.entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, entry)| entry.enabled && entry.prompt_visible)
|
||||
.map(|(id, entry)| SkillSelectionDocument {
|
||||
id,
|
||||
name: entry.name.as_str(),
|
||||
short_description: entry.short_description.as_deref(),
|
||||
description: entry.description.as_str(),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let eligible_ids = documents
|
||||
.iter()
|
||||
.map(|document| document.id)
|
||||
.collect::<HashSet<_>>();
|
||||
let mut ranked_selections = Vec::with_capacity(self.selectors.len());
|
||||
|
||||
for selector in &self.selectors {
|
||||
let start = Instant::now();
|
||||
let selection =
|
||||
selector.select(&query.text, &documents, /*limit*/ MAX_SHADOW_RESULTS);
|
||||
let duration = start.elapsed();
|
||||
let selected_ids = sanitize_selected_ids(&selection, &eligible_ids);
|
||||
self.record_metrics(ShadowSelectionObservation {
|
||||
method: selector.method(),
|
||||
selection: &selection,
|
||||
query_truncated_before_selection: query.truncated,
|
||||
query_script,
|
||||
catalog_entry_count: documents.len(),
|
||||
selected_entry_count: selected_ids.len(),
|
||||
duration,
|
||||
});
|
||||
ranked_selections.push(RankedSelection {
|
||||
method: selector.method(),
|
||||
skill_resources: selected_ids
|
||||
.iter()
|
||||
.map(|id| normalize_skill_resource(catalog.entries[*id].main_prompt.as_str()))
|
||||
.collect(),
|
||||
});
|
||||
tracing::debug!(
|
||||
method = selector.method(),
|
||||
catalog_entries = documents.len(),
|
||||
selected_entries = selected_ids.len(),
|
||||
query_terms = selection.query_term_count,
|
||||
query_script,
|
||||
query_truncated = query.truncated || selection.query_truncated,
|
||||
candidate_set_truncated = selection.candidate_set_truncated,
|
||||
"ran shadow skill selection"
|
||||
);
|
||||
}
|
||||
|
||||
ShadowSelectionTurnState {
|
||||
ranked_selections,
|
||||
query_script,
|
||||
seen_skill_resources: Mutex::new(HashSet::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_invocation(&self, state: &ShadowSelectionTurnState, skill_resource: &str) {
|
||||
let skill_resource = normalize_skill_resource(skill_resource);
|
||||
if !state
|
||||
.seen_skill_resources
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.insert(skill_resource.clone())
|
||||
{
|
||||
return;
|
||||
}
|
||||
let Some(metrics_client) = self.metrics_client.as_ref() else {
|
||||
return;
|
||||
};
|
||||
for selection in &state.ranked_selections {
|
||||
let rank = selection
|
||||
.skill_resources
|
||||
.iter()
|
||||
.position(|candidate| candidate == &skill_resource)
|
||||
.map(|index| index + 1);
|
||||
let tags = [
|
||||
("method", selection.method),
|
||||
("hit", bool_tag(rank.is_some())),
|
||||
("rank", rank_bucket(rank)),
|
||||
("query_script", state.query_script),
|
||||
];
|
||||
let _ = metrics_client.counter(INVOCATION_METRIC, /*inc*/ 1, &tags);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_metrics(&self, observation: ShadowSelectionObservation<'_>) {
|
||||
let Some(metrics_client) = self.metrics_client.as_ref() else {
|
||||
return;
|
||||
};
|
||||
let ShadowSelectionObservation {
|
||||
method,
|
||||
selection,
|
||||
query_truncated_before_selection,
|
||||
query_script,
|
||||
catalog_entry_count,
|
||||
selected_entry_count,
|
||||
duration,
|
||||
} = observation;
|
||||
let status = selection_status(selection, selected_entry_count);
|
||||
let query_truncated =
|
||||
bool_tag(query_truncated_before_selection || selection.query_truncated);
|
||||
let candidate_set_truncated = bool_tag(selection.candidate_set_truncated);
|
||||
let tags = [
|
||||
("method", method),
|
||||
("status", status),
|
||||
("query_script", query_script),
|
||||
("query_truncated", query_truncated),
|
||||
("candidate_set_truncated", candidate_set_truncated),
|
||||
];
|
||||
let _ = metrics_client.counter(RUN_METRIC, /*inc*/ 1, &tags);
|
||||
let _ = metrics_client.record_duration(DURATION_METRIC, duration, &tags);
|
||||
let _ = metrics_client.histogram(
|
||||
CATALOG_ENTRY_COUNT_METRIC,
|
||||
metric_value(catalog_entry_count),
|
||||
&tags,
|
||||
);
|
||||
let _ = metrics_client.histogram(
|
||||
SELECTED_ENTRY_COUNT_METRIC,
|
||||
metric_value(selected_entry_count),
|
||||
&tags,
|
||||
);
|
||||
let _ = metrics_client.histogram(
|
||||
QUERY_TERM_COUNT_METRIC,
|
||||
metric_value(selection.query_term_count),
|
||||
&tags,
|
||||
);
|
||||
let _ = metrics_client.histogram(
|
||||
REDUCTION_BPS_METRIC,
|
||||
reduction_bps(catalog_entry_count, selected_entry_count),
|
||||
&tags,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct ShadowSelectionTurnState {
|
||||
ranked_selections: Vec<RankedSelection>,
|
||||
query_script: &'static str,
|
||||
seen_skill_resources: Mutex<HashSet<String>>,
|
||||
}
|
||||
|
||||
struct RankedSelection {
|
||||
method: &'static str,
|
||||
skill_resources: Vec<String>,
|
||||
}
|
||||
|
||||
struct ShadowSelectionObservation<'a> {
|
||||
method: &'static str,
|
||||
selection: &'a CheapSkillSelection,
|
||||
query_truncated_before_selection: bool,
|
||||
query_script: &'static str,
|
||||
catalog_entry_count: usize,
|
||||
selected_entry_count: usize,
|
||||
duration: Duration,
|
||||
}
|
||||
|
||||
fn sanitize_selected_ids(
|
||||
selection: &CheapSkillSelection,
|
||||
eligible_ids: &HashSet<usize>,
|
||||
) -> Vec<usize> {
|
||||
let mut seen = HashSet::new();
|
||||
selection
|
||||
.candidate_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| eligible_ids.contains(id) && seen.insert(*id))
|
||||
.take(MAX_SHADOW_RESULTS)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn selection_status(selection: &CheapSkillSelection, selected_entry_count: usize) -> &'static str {
|
||||
if selection.query_term_count == 0 {
|
||||
"no_query_terms"
|
||||
} else if selected_entry_count == 0 {
|
||||
"no_matches"
|
||||
} else {
|
||||
"selected"
|
||||
}
|
||||
}
|
||||
|
||||
fn reduction_bps(catalog_entry_count: usize, selected_entry_count: usize) -> i64 {
|
||||
if catalog_entry_count == 0 {
|
||||
return 0;
|
||||
}
|
||||
10_000i64.saturating_sub(ratio_bps(selected_entry_count, catalog_entry_count))
|
||||
}
|
||||
|
||||
fn ratio_bps(numerator: usize, denominator: usize) -> i64 {
|
||||
if denominator == 0 {
|
||||
return 0;
|
||||
}
|
||||
let numerator = u128::try_from(numerator).unwrap_or(u128::MAX);
|
||||
let denominator = u128::try_from(denominator).unwrap_or(u128::MAX);
|
||||
let basis_points = numerator.saturating_mul(10_000) / denominator;
|
||||
i64::try_from(basis_points).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn metric_value(value: usize) -> i64 {
|
||||
i64::try_from(value).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
fn bool_tag(value: bool) -> &'static str {
|
||||
if value { "true" } else { "false" }
|
||||
}
|
||||
|
||||
fn rank_bucket(rank: Option<usize>) -> &'static str {
|
||||
match rank {
|
||||
Some(1) => "1",
|
||||
Some(2..=5) => "2_5",
|
||||
Some(6..=10) => "6_10",
|
||||
Some(11..=MAX_SHADOW_RESULTS) => "11_20",
|
||||
Some(_) | None => "miss",
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_skill_resource(skill_resource: &str) -> String {
|
||||
skill_resource.replace('\\', "/")
|
||||
}
|
||||
|
||||
fn query_script_tag(query: &str) -> &'static str {
|
||||
let mut has_ascii_latin = false;
|
||||
let mut has_cjk = false;
|
||||
let mut has_other = false;
|
||||
|
||||
for character in query.chars().filter(|character| character.is_alphabetic()) {
|
||||
if character.is_ascii_alphabetic() {
|
||||
has_ascii_latin = true;
|
||||
} else if is_cjk(character) {
|
||||
has_cjk = true;
|
||||
} else {
|
||||
has_other = true;
|
||||
}
|
||||
}
|
||||
|
||||
match (has_ascii_latin, has_cjk, has_other) {
|
||||
(false, false, false) => "none",
|
||||
(true, false, false) => "ascii_latin",
|
||||
(false, true, false) => "cjk",
|
||||
(false, false, true) => "other",
|
||||
(true, true, false) | (true, false, true) | (false, true, true) | (true, true, true) => {
|
||||
"mixed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_cjk(character: char) -> bool {
|
||||
matches!(
|
||||
character,
|
||||
'\u{1100}'..='\u{11ff}'
|
||||
| '\u{3040}'..='\u{30ff}'
|
||||
| '\u{3100}'..='\u{312f}'
|
||||
| '\u{3130}'..='\u{318f}'
|
||||
| '\u{31a0}'..='\u{31bf}'
|
||||
| '\u{31f0}'..='\u{31ff}'
|
||||
| '\u{3400}'..='\u{4dbf}'
|
||||
| '\u{4e00}'..='\u{9fff}'
|
||||
| '\u{a960}'..='\u{a97f}'
|
||||
| '\u{ac00}'..='\u{d7af}'
|
||||
| '\u{d7b0}'..='\u{d7ff}'
|
||||
| '\u{f900}'..='\u{faff}'
|
||||
| '\u{20000}'..='\u{2fa1f}'
|
||||
)
|
||||
}
|
||||
|
||||
struct ShadowQuery {
|
||||
text: String,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
fn build_shadow_query(inputs: &[UserInput]) -> ShadowQuery {
|
||||
let mut text = String::new();
|
||||
let mut truncated = false;
|
||||
for input in inputs {
|
||||
let part = match input {
|
||||
UserInput::Text { text, .. } => text.as_str(),
|
||||
UserInput::Skill { name, .. } | UserInput::Mention { name, .. } => name.as_str(),
|
||||
_ => continue,
|
||||
};
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !text.is_empty() && !push_bounded(&mut text, " ") {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
if !push_bounded(&mut text, part) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ShadowQuery { text, truncated }
|
||||
}
|
||||
|
||||
fn push_bounded(destination: &mut String, value: &str) -> bool {
|
||||
let remaining = MAX_SHADOW_QUERY_BYTES.saturating_sub(destination.len());
|
||||
if value.len() <= remaining {
|
||||
destination.push_str(value);
|
||||
return true;
|
||||
}
|
||||
let mut end = remaining;
|
||||
while !value.is_char_boundary(end) {
|
||||
end = end.saturating_sub(1);
|
||||
}
|
||||
destination.push_str(&value[..end]);
|
||||
false
|
||||
}
|
||||
@@ -20,6 +20,7 @@ use crate::catalog::SkillResourceId;
|
||||
use crate::catalog::SkillSourceKind;
|
||||
use crate::provider::SkillListQuery;
|
||||
use crate::provider::SkillReadRequest;
|
||||
use crate::shadow_selection_experiment::ShadowSelectionTurnState;
|
||||
use crate::sources::SkillProviders;
|
||||
|
||||
const MAX_CACHED_ORCHESTRATOR_RESOURCES: usize = 100;
|
||||
@@ -30,6 +31,7 @@ pub(crate) struct SkillsThreadState {
|
||||
orchestrator_skills_available: bool,
|
||||
executor_cache: Mutex<Vec<CachedExecutorCatalog>>,
|
||||
orchestrator_cache: Mutex<Option<Arc<OrchestratorGenerationCache>>>,
|
||||
shadow_selection_turn: Mutex<Option<ShadowSelectionTurn>>,
|
||||
}
|
||||
|
||||
impl SkillsThreadState {
|
||||
@@ -39,6 +41,7 @@ impl SkillsThreadState {
|
||||
orchestrator_skills_available,
|
||||
executor_cache: Mutex::new(Vec::new()),
|
||||
orchestrator_cache: Mutex::new(None),
|
||||
shadow_selection_turn: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,6 +63,33 @@ impl SkillsThreadState {
|
||||
self.orchestrator_skills_available && self.config().orchestrator_skills_enabled
|
||||
}
|
||||
|
||||
pub(crate) fn replace_shadow_selection_turn(
|
||||
&self,
|
||||
turn_id: String,
|
||||
state: Option<ShadowSelectionTurnState>,
|
||||
) {
|
||||
*self
|
||||
.shadow_selection_turn
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||||
state.map(|state| ShadowSelectionTurn {
|
||||
turn_id,
|
||||
state: Arc::new(state),
|
||||
});
|
||||
}
|
||||
|
||||
pub(crate) fn shadow_selection_turn(
|
||||
&self,
|
||||
turn_id: &str,
|
||||
) -> Option<Arc<ShadowSelectionTurnState>> {
|
||||
self.shadow_selection_turn
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.as_ref()
|
||||
.filter(|turn| turn.turn_id == turn_id)
|
||||
.map(|turn| Arc::clone(&turn.state))
|
||||
}
|
||||
|
||||
/// Returns catalogs for stable selected roots.
|
||||
///
|
||||
/// The first catalog returned for a root remains cached until this thread state is dropped.
|
||||
@@ -196,6 +226,11 @@ impl SkillsThreadState {
|
||||
}
|
||||
}
|
||||
|
||||
struct ShadowSelectionTurn {
|
||||
turn_id: String,
|
||||
state: Arc<ShadowSelectionTurnState>,
|
||||
}
|
||||
|
||||
struct CachedExecutorCatalog {
|
||||
root: SelectedCapabilityRoot,
|
||||
catalog: SkillCatalog,
|
||||
|
||||
@@ -23,6 +23,7 @@ use crate::catalog::SkillAuthority;
|
||||
use crate::catalog::SkillCatalog;
|
||||
use crate::catalog::SkillSourceKind;
|
||||
use crate::provider::SkillListQuery;
|
||||
use crate::shadow_selection_experiment::ShadowSelectionExperiment;
|
||||
use crate::sources::SkillProviders;
|
||||
use crate::state::SkillsThreadState;
|
||||
|
||||
@@ -37,11 +38,13 @@ pub(crate) fn skill_tools(
|
||||
providers: SkillProviders,
|
||||
mcp_resources: Option<Arc<McpResourceClient>>,
|
||||
thread_state: Arc<SkillsThreadState>,
|
||||
shadow_selection: Arc<ShadowSelectionExperiment>,
|
||||
) -> Vec<Arc<dyn ToolExecutor<ToolCall>>> {
|
||||
let context = SkillToolContext {
|
||||
providers,
|
||||
mcp_resources,
|
||||
thread_state,
|
||||
shadow_selection,
|
||||
};
|
||||
vec![
|
||||
Arc::new(list::ListTool {
|
||||
@@ -56,6 +59,7 @@ struct SkillToolContext {
|
||||
providers: SkillProviders,
|
||||
mcp_resources: Option<Arc<McpResourceClient>>,
|
||||
thread_state: Arc<SkillsThreadState>,
|
||||
shadow_selection: Arc<ShadowSelectionExperiment>,
|
||||
}
|
||||
|
||||
impl SkillToolContext {
|
||||
|
||||
@@ -63,14 +63,14 @@ impl ToolExecutor<ToolCall> for ReadTool {
|
||||
validate_handle("resource", &args.resource, MAX_HANDLE_BYTES)?;
|
||||
|
||||
let catalog = self.context.catalog(&call.turn_id, args.authority).await;
|
||||
let package_is_available = catalog.entries.iter().any(|entry| {
|
||||
let Some(skill_entry) = catalog.entries.iter().find(|entry| {
|
||||
entry.enabled && entry.authority == authority && entry.id.0 == args.package
|
||||
});
|
||||
if !package_is_available {
|
||||
}) else {
|
||||
return Err(FunctionCallError::RespondToModel(
|
||||
"skill package is not available from the requested authority".to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let main_prompt = skill_entry.main_prompt.clone();
|
||||
|
||||
let requested_resource = SkillResourceId::new(args.resource);
|
||||
let result = self
|
||||
@@ -103,6 +103,16 @@ impl ToolExecutor<ToolCall> for ReadTool {
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(state) = self
|
||||
.context
|
||||
.thread_state
|
||||
.shadow_selection_turn(&call.turn_id)
|
||||
{
|
||||
self.context
|
||||
.shadow_selection
|
||||
.record_invocation(&state, main_prompt.as_str());
|
||||
}
|
||||
|
||||
external_json_output(&ReadResponse {
|
||||
resource: result.resource.as_str().to_string(),
|
||||
contents: result.contents,
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use codex_core_skills::HostSkillsSnapshot;
|
||||
use codex_core_skills::SkillLoadOutcome;
|
||||
use codex_core_skills::SkillMetadata;
|
||||
use codex_extension_api::ConversationHistory;
|
||||
use codex_extension_api::ExtensionData;
|
||||
use codex_extension_api::ExtensionRegistry;
|
||||
use codex_extension_api::ExtensionRegistryBuilder;
|
||||
use codex_extension_api::NoopTurnItemEmitter;
|
||||
use codex_extension_api::SkillInvocationInput;
|
||||
use codex_extension_api::SkillInvocationKind;
|
||||
use codex_extension_api::ThreadStartInput;
|
||||
use codex_extension_api::ToolCall;
|
||||
use codex_extension_api::ToolPayload;
|
||||
use codex_extension_api::TurnInputContext;
|
||||
use codex_mcp::CODEX_APPS_MCP_SERVER_NAME;
|
||||
use codex_otel::MetricsClient;
|
||||
use codex_otel::MetricsConfig;
|
||||
use codex_protocol::protocol::SessionSource;
|
||||
use codex_protocol::protocol::SkillScope;
|
||||
use codex_protocol::protocol::TruncationPolicy;
|
||||
use codex_protocol::user_input::UserInput;
|
||||
use codex_skills_extension::SkillProviders;
|
||||
use codex_skills_extension::SkillsExtensionConfig;
|
||||
use codex_skills_extension::catalog::SkillAuthority;
|
||||
use codex_skills_extension::catalog::SkillCatalog;
|
||||
use codex_skills_extension::catalog::SkillCatalogEntry;
|
||||
use codex_skills_extension::catalog::SkillPackageId;
|
||||
use codex_skills_extension::catalog::SkillReadResult;
|
||||
use codex_skills_extension::catalog::SkillResourceId;
|
||||
use codex_skills_extension::catalog::SkillSearchResult;
|
||||
use codex_skills_extension::catalog::SkillSourceKind;
|
||||
use codex_skills_extension::install_with_providers_and_metrics;
|
||||
use codex_skills_extension::provider::SkillListQuery;
|
||||
use codex_skills_extension::provider::SkillProvider;
|
||||
use codex_skills_extension::provider::SkillProviderFuture;
|
||||
use codex_skills_extension::provider::SkillReadRequest;
|
||||
use codex_skills_extension::provider::SkillSearchRequest;
|
||||
use codex_utils_absolute_path::AbsolutePathBuf;
|
||||
use opentelemetry_sdk::metrics::InMemoryMetricExporter;
|
||||
use opentelemetry_sdk::metrics::data::AggregatedMetrics;
|
||||
use opentelemetry_sdk::metrics::data::MetricData;
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error>>;
|
||||
type InvocationPoint = (BTreeMap<String, String>, u64);
|
||||
|
||||
const INVOCATION_METRIC: &str = "codex.skills.shadow_selection.invocation";
|
||||
const PACKAGE: &str = "orchestrator/demo";
|
||||
const MAIN_RESOURCE: &str = "skill://orchestrator/demo/SKILL.md";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct OrchestratorProvider;
|
||||
|
||||
impl SkillProvider for OrchestratorProvider {
|
||||
fn list(&self, _query: SkillListQuery) -> SkillProviderFuture<'_, SkillCatalog> {
|
||||
Box::pin(std::future::ready(Ok(SkillCatalog {
|
||||
entries: vec![SkillCatalogEntry::new(
|
||||
SkillPackageId(PACKAGE.to_string()),
|
||||
SkillAuthority::new(SkillSourceKind::Orchestrator, CODEX_APPS_MCP_SERVER_NAME),
|
||||
"demo",
|
||||
"Use the demo skill.",
|
||||
SkillResourceId::new(MAIN_RESOURCE),
|
||||
)],
|
||||
warnings: Vec::new(),
|
||||
})))
|
||||
}
|
||||
|
||||
fn read(&self, request: SkillReadRequest) -> SkillProviderFuture<'_, SkillReadResult> {
|
||||
Box::pin(std::future::ready(Ok(SkillReadResult {
|
||||
resource: request.resource,
|
||||
contents: "# Demo\n\nUse the demo skill.".to_string(),
|
||||
})))
|
||||
}
|
||||
|
||||
fn search(&self, _request: SkillSearchRequest) -> SkillProviderFuture<'_, SkillSearchResult> {
|
||||
Box::pin(std::future::ready(Ok(SkillSearchResult::default())))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn implicit_core_and_native_read_invocations_share_turn_local_recording() -> TestResult {
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory(
|
||||
"test",
|
||||
"codex-skills-extension",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
InMemoryMetricExporter::default(),
|
||||
)
|
||||
.with_runtime_reader(),
|
||||
)?;
|
||||
let mut builder = ExtensionRegistryBuilder::<()>::new();
|
||||
install_with_providers_and_metrics(
|
||||
&mut builder,
|
||||
SkillProviders::new().with_orchestrator_provider(Arc::new(OrchestratorProvider)),
|
||||
Some(metrics.clone()),
|
||||
|_| SkillsExtensionConfig {
|
||||
include_instructions: false,
|
||||
bundled_skills_enabled: false,
|
||||
orchestrator_skills_enabled: true,
|
||||
shadow_selection_enabled: true,
|
||||
},
|
||||
);
|
||||
let registry = builder.build();
|
||||
let session_store = ExtensionData::new("session");
|
||||
let thread_store = ExtensionData::new("thread");
|
||||
registry.thread_lifecycle_contributors()[0]
|
||||
.on_thread_start(ThreadStartInput {
|
||||
config: &(),
|
||||
session_source: &SessionSource::Cli,
|
||||
persistent_thread_state_available: true,
|
||||
environments: &[],
|
||||
session_store: &session_store,
|
||||
thread_store: &thread_store,
|
||||
})
|
||||
.await;
|
||||
|
||||
let core_turn = start_turn(®istry, &session_store, &thread_store, "turn-core").await;
|
||||
registry.skill_invocation_contributors()[0]
|
||||
.on_skill_invocation(SkillInvocationInput {
|
||||
session_store: &session_store,
|
||||
thread_store: &thread_store,
|
||||
turn_store: core_turn.as_ref(),
|
||||
turn_id: "turn-core",
|
||||
skill_resource: MAIN_RESOURCE,
|
||||
kind: SkillInvocationKind::Implicit,
|
||||
})
|
||||
.await;
|
||||
|
||||
start_turn(®istry, &session_store, &thread_store, "turn-tool").await;
|
||||
let tools = registry.tool_contributors()[0].tools(&session_store, &thread_store);
|
||||
let read_tool = tools
|
||||
.iter()
|
||||
.find(|tool| tool.tool_name().name == "read")
|
||||
.ok_or("skills.read tool should be registered")?;
|
||||
for call_id in ["call-1", "call-2"] {
|
||||
read_tool
|
||||
.handle(ToolCall {
|
||||
turn_id: "turn-tool".to_string(),
|
||||
call_id: call_id.to_string(),
|
||||
tool_name: read_tool.tool_name(),
|
||||
model: "gpt-test".to_string(),
|
||||
truncation_policy: TruncationPolicy::Bytes(1_024),
|
||||
conversation_history: ConversationHistory::default(),
|
||||
turn_item_emitter: Arc::new(NoopTurnItemEmitter),
|
||||
environments: Vec::new(),
|
||||
payload: ToolPayload::Function {
|
||||
arguments: serde_json::json!({
|
||||
"authority": {"kind": "orchestrator"},
|
||||
"package": PACKAGE,
|
||||
"resource": MAIN_RESOURCE,
|
||||
})
|
||||
.to_string(),
|
||||
},
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
vec![(
|
||||
BTreeMap::from([
|
||||
("hit".to_string(), "true".to_string()),
|
||||
("method".to_string(), "weighted_lexical_v1".to_string()),
|
||||
("query_script".to_string(), "ascii_latin".to_string()),
|
||||
("rank".to_string(), "1".to_string()),
|
||||
]),
|
||||
2,
|
||||
)],
|
||||
invocation_points(&metrics)?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn host_snapshot_is_available_only_to_shadow_selection() -> TestResult {
|
||||
let metrics = MetricsClient::new(
|
||||
MetricsConfig::in_memory(
|
||||
"test",
|
||||
"codex-skills-extension",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
InMemoryMetricExporter::default(),
|
||||
)
|
||||
.with_runtime_reader(),
|
||||
)?;
|
||||
let mut builder = ExtensionRegistryBuilder::<()>::new();
|
||||
install_with_providers_and_metrics(
|
||||
&mut builder,
|
||||
SkillProviders::new(),
|
||||
Some(metrics.clone()),
|
||||
|_| SkillsExtensionConfig {
|
||||
include_instructions: true,
|
||||
bundled_skills_enabled: false,
|
||||
orchestrator_skills_enabled: false,
|
||||
shadow_selection_enabled: true,
|
||||
},
|
||||
);
|
||||
let registry = builder.build();
|
||||
let session_store = ExtensionData::new("session");
|
||||
let thread_store = ExtensionData::new("thread");
|
||||
registry.thread_lifecycle_contributors()[0]
|
||||
.on_thread_start(ThreadStartInput {
|
||||
config: &(),
|
||||
session_source: &SessionSource::Cli,
|
||||
persistent_thread_state_available: true,
|
||||
environments: &[],
|
||||
session_store: &session_store,
|
||||
thread_store: &thread_store,
|
||||
})
|
||||
.await;
|
||||
|
||||
let skill_path = AbsolutePathBuf::try_from(
|
||||
std::env::temp_dir()
|
||||
.join("codex-shadow-host-skill")
|
||||
.join("SKILL.md"),
|
||||
)?;
|
||||
let skill_resource = skill_path.to_string_lossy().into_owned();
|
||||
let mut outcome = SkillLoadOutcome::default();
|
||||
outcome.skills.push(SkillMetadata {
|
||||
name: "演示文稿".to_string(),
|
||||
description: "创建演示文稿。".to_string(),
|
||||
short_description: None,
|
||||
interface: None,
|
||||
dependencies: None,
|
||||
policy: None,
|
||||
path_to_skills_md: skill_path,
|
||||
scope: SkillScope::User,
|
||||
plugin_id: None,
|
||||
});
|
||||
let turn_store = ExtensionData::new("turn-host");
|
||||
turn_store.insert(HostSkillsSnapshot::new(Arc::new(outcome)));
|
||||
|
||||
let fragments = registry.turn_input_contributors()[0]
|
||||
.contribute(
|
||||
TurnInputContext {
|
||||
turn_id: "turn-host".to_string(),
|
||||
user_input: vec![UserInput::Text {
|
||||
text: "演示文稿".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
environments: Vec::new(),
|
||||
},
|
||||
&session_store,
|
||||
&thread_store,
|
||||
&turn_store,
|
||||
)
|
||||
.await;
|
||||
assert!(fragments.is_empty());
|
||||
|
||||
registry.skill_invocation_contributors()[0]
|
||||
.on_skill_invocation(SkillInvocationInput {
|
||||
session_store: &session_store,
|
||||
thread_store: &thread_store,
|
||||
turn_store: &turn_store,
|
||||
turn_id: "turn-host",
|
||||
skill_resource: &skill_resource,
|
||||
kind: SkillInvocationKind::Implicit,
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
vec![(
|
||||
BTreeMap::from([
|
||||
("hit".to_string(), "true".to_string()),
|
||||
("method".to_string(), "weighted_lexical_v1".to_string()),
|
||||
("query_script".to_string(), "cjk".to_string()),
|
||||
("rank".to_string(), "1".to_string()),
|
||||
]),
|
||||
1,
|
||||
)],
|
||||
invocation_points(&metrics)?,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start_turn(
|
||||
registry: &ExtensionRegistry<()>,
|
||||
session_store: &ExtensionData,
|
||||
thread_store: &ExtensionData,
|
||||
turn_id: &str,
|
||||
) -> Arc<ExtensionData> {
|
||||
let turn_store = Arc::new(ExtensionData::new(turn_id));
|
||||
registry.turn_input_contributors()[0]
|
||||
.contribute(
|
||||
TurnInputContext {
|
||||
turn_id: turn_id.to_string(),
|
||||
user_input: vec![UserInput::Text {
|
||||
text: "use demo".to_string(),
|
||||
text_elements: Vec::new(),
|
||||
}],
|
||||
environments: Vec::new(),
|
||||
},
|
||||
session_store,
|
||||
thread_store,
|
||||
turn_store.as_ref(),
|
||||
)
|
||||
.await;
|
||||
turn_store
|
||||
}
|
||||
|
||||
fn invocation_points(
|
||||
metrics: &MetricsClient,
|
||||
) -> Result<Vec<InvocationPoint>, Box<dyn std::error::Error>> {
|
||||
let snapshot = metrics.snapshot()?;
|
||||
let metric = snapshot
|
||||
.scope_metrics()
|
||||
.flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics)
|
||||
.find(|metric| metric.name() == INVOCATION_METRIC)
|
||||
.ok_or("invocation metric was not emitted")?;
|
||||
match metric.data() {
|
||||
AggregatedMetrics::U64(MetricData::Sum(sum)) => Ok(sum
|
||||
.data_points()
|
||||
.map(|point| {
|
||||
(
|
||||
point
|
||||
.attributes()
|
||||
.map(|attribute| {
|
||||
(
|
||||
attribute.key.as_str().to_string(),
|
||||
attribute.value.as_str().to_string(),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
point.value(),
|
||||
)
|
||||
})
|
||||
.collect()),
|
||||
other => Err(format!("unexpected invocation metric data: {other:?}").into()),
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,8 @@ async fn installed_extension_uses_host_service_snapshot() -> TestResult {
|
||||
.ok_or("skill path should have a parent")?,
|
||||
)?;
|
||||
std::fs::write(&skill_path, DEMO_SKILL_CONTENTS)?;
|
||||
let config = default_config();
|
||||
let mut config = default_config();
|
||||
config.shadow_selection_enabled = true;
|
||||
|
||||
let mut builder = ExtensionRegistryBuilder::new();
|
||||
install(&mut builder, skills_extension_config);
|
||||
@@ -781,6 +782,7 @@ struct TestConfig {
|
||||
include_instructions: bool,
|
||||
bundled_skills_enabled: bool,
|
||||
orchestrator_skills_enabled: bool,
|
||||
shadow_selection_enabled: bool,
|
||||
}
|
||||
|
||||
fn default_config() -> TestConfig {
|
||||
@@ -788,6 +790,7 @@ fn default_config() -> TestConfig {
|
||||
include_instructions: true,
|
||||
bundled_skills_enabled: true,
|
||||
orchestrator_skills_enabled: true,
|
||||
shadow_selection_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -796,6 +799,7 @@ fn skills_extension_config(config: &TestConfig) -> SkillsExtensionConfig {
|
||||
include_instructions: config.include_instructions,
|
||||
bundled_skills_enabled: config.bundled_skills_enabled,
|
||||
orchestrator_skills_enabled: config.orchestrator_skills_enabled,
|
||||
shadow_selection_enabled: config.shadow_selection_enabled,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -206,6 +206,8 @@ pub enum Feature {
|
||||
ConcurrentReasoningSummaries,
|
||||
/// Allow prompting and installing missing MCP dependencies.
|
||||
SkillMcpDependencyInstall,
|
||||
/// Run cheap skill-search methods in shadow mode and emit experiment metrics.
|
||||
SkillSearch,
|
||||
/// Removed compatibility flag for deleted skill env var dependency prompting.
|
||||
SkillEnvVarDependencyPrompt,
|
||||
/// Enable the unified mention popup used by default in the TUI.
|
||||
@@ -1190,6 +1192,12 @@ pub const FEATURES: &[FeatureSpec] = &[
|
||||
stage: Stage::Stable,
|
||||
default_enabled: true,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::SkillSearch,
|
||||
key: "skill_search",
|
||||
stage: Stage::UnderDevelopment,
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
id: Feature::SkillEnvVarDependencyPrompt,
|
||||
key: "skill_env_var_dependency_prompt",
|
||||
|
||||
Reference in New Issue
Block a user