Evaluate character matching over skill routing metadata (#35663)

## What changed

- Add a character n-gram shadow selector that combines skill descriptions with host interface metadata and tool dependency names and descriptions.
- Bound routing fields, dependency records, and candidate counts before ranking.
- Exclude explicitly selected skills from shadow candidates and only record invocations for skills eligible for the experiment.

## Testing

- Add unit coverage for host interface metadata, connector dependencies, baseline description matches, and long routing metadata.

GitOrigin-RevId: 36ef0cb4a68601f2d9d41c3344c88f7927c32bf4

codex-rs/ext/skills/src/dynamic_skill_selector.rs

@@ -1,10 +1,12 @@
 mod character_ngram;
+mod character_routing_card;
 mod fielded_bm25;
 mod multi_query_lexical;
 mod routing_card_lexical;
 mod rrf_lexical_char;
 mod weighted_lexical;
 pub(crate) use character_ngram::CharacterNgramSkillSelector;
+pub(crate) use character_routing_card::CharacterRoutingCardSkillSelector;
 use codex_skills::SkillDependencies;
 pub(crate) use fielded_bm25::FieldedBm25SkillSelector;
 pub(crate) use multi_query_lexical::MultiQueryLexicalSkillSelector;

codex-rs/ext/skills/src/dynamic_skill_selector/character_routing_card.rs

@@ -0,0 +1,131 @@
+use std::collections::HashMap;
+
+use codex_core_skills::HostSkillsSnapshot;
+
+use crate::catalog::SkillCatalog;
+use crate::catalog::SkillSourceKind;
+
+use super::CharacterNgramSkillSelector;
+use super::CheapSkillSelection;
+use super::CheapSkillSelector;
+use super::SkillSelectionDocument;
+
+const MAX_ROUTING_FIELD_BYTES: usize = 4 * 1024;
+const MAX_DEPENDENCY_RECORDS: usize = 32;
+const MAX_CANDIDATES: usize = 1_000;
+
+#[derive(Clone, Debug, Default)]
+pub(crate) struct CharacterRoutingCardSkillSelector {
+    routing_fields: HashMap<usize, String>,
+}
+
+impl CharacterRoutingCardSkillSelector {
+    pub(crate) fn new(catalog: &SkillCatalog, host_snapshot: Option<&HostSkillsSnapshot>) -> Self {
+        let host_skill_ids = catalog
+            .entries
+            .iter()
+            .enumerate()
+            .filter(|(_, entry)| {
+                entry.authority.kind == SkillSourceKind::Host && entry.is_model_visible()
+            })
+            .take(MAX_CANDIDATES)
+            .map(|(id, entry)| (entry.main_prompt.as_str().replace('\\', "/"), id))
+            .collect::<HashMap<_, _>>();
+        let routing_fields = host_snapshot
+            .into_iter()
+            .flat_map(|snapshot| snapshot.outcome().skills.iter())
+            .filter_map(|skill| {
+                let id = host_skill_ids
+                    .get(&skill.path_to_skills_md.to_string_lossy().replace('\\', "/"))?;
+                let interface = skill.interface.as_ref()?;
+                let mut field = String::new();
+                for value in [
+                    interface.display_name.as_deref(),
+                    interface.short_description.as_deref(),
+                    interface.default_prompt.as_deref(),
+                ]
+                .into_iter()
+                .flatten()
+                {
+                    push_bounded(&mut field, value);
+                }
+                (!field.is_empty()).then_some((*id, field))
+            })
+            .take(MAX_CANDIDATES)
+            .collect();
+        Self { routing_fields }
+    }
+}
+
+impl CheapSkillSelector for CharacterRoutingCardSkillSelector {
+    fn method(&self) -> &'static str {
+        "character_routing_card_v1"
+    }
+
+    fn select(
+        &self,
+        query: &str,
+        documents: &[SkillSelectionDocument<'_>],
+        limit: usize,
+    ) -> CheapSkillSelection {
+        if limit == 0 {
+            return CharacterNgramSkillSelector.select(query, documents, /*limit*/ 0);
+        }
+
+        let routing_fields = documents
+            .iter()
+            .take(MAX_CANDIDATES)
+            .map(|document| {
+                let mut field = String::new();
+                if let Some(description) = document.short_description {
+                    push_bounded(&mut field, description);
+                }
+                if let Some(routing_field) = self.routing_fields.get(&document.id) {
+                    push_bounded(&mut field, routing_field);
+                }
+                if let Some(dependencies) = document.dependencies {
+                    for dependency in dependencies.tools.iter().take(MAX_DEPENDENCY_RECORDS) {
+                        push_bounded(&mut field, dependency.value.as_str());
+                        if let Some(description) = dependency.description.as_deref() {
+                            push_bounded(&mut field, description);
+                        }
+                    }
+                }
+                field
+            })
+            .collect::<Vec<_>>();
+        let routing_documents = documents
+            .iter()
+            .zip(&routing_fields)
+            .map(|(document, field)| SkillSelectionDocument {
+                short_description: (!field.is_empty()).then_some(field.as_str()),
+                ..*document
+            })
+            .collect::<Vec<_>>();
+        let selection = CharacterNgramSkillSelector.select(query, &routing_documents, limit);
+        CheapSkillSelection {
+            candidate_set_truncated: documents.len() > MAX_CANDIDATES
+                || selection.candidate_set_truncated,
+            ..selection
+        }
+    }
+}
+
+fn push_bounded(destination: &mut String, value: &str) {
+    if value.is_empty() || destination.len() == MAX_ROUTING_FIELD_BYTES {
+        return;
+    }
+    if !destination.is_empty() {
+        destination.push(' ');
+    }
+    let remaining = MAX_ROUTING_FIELD_BYTES.saturating_sub(destination.len());
+    let mut end = value.len().min(remaining);
+    while !value.is_char_boundary(end) {
+        end -= 1;
+    }
+    destination.push_str(&value[..end]);
+}
+
+#[cfg(test)]
+#[path = "character_routing_card_tests.rs"]
+mod tests;

codex-rs/ext/skills/src/dynamic_skill_selector/character_routing_card_tests.rs

@@ -0,0 +1,152 @@
+use codex_core_skills::SkillLoadOutcome;
+use codex_protocol::protocol::SkillScope;
+use codex_skills::SkillDependencies;
+use codex_skills::SkillInterface;
+use codex_skills::SkillMetadata;
+use codex_skills::SkillToolDependency;
+use codex_utils_absolute_path::AbsolutePathBuf;
+use pretty_assertions::assert_eq;
+
+use crate::catalog::SkillAuthority;
+use crate::catalog::SkillCatalogEntry;
+use crate::catalog::SkillPackageId;
+use crate::catalog::SkillResourceId;
+
+use super::*;
+
+#[test]
+fn host_interface_metadata_makes_unlisted_workflow_discoverable() {
+    let path = AbsolutePathBuf::try_from(std::env::temp_dir().join("routing-card/SKILL.md"))
+        .expect("temporary directory should be absolute");
+    let resource = path.to_string_lossy().to_string();
+    let mut outcome = SkillLoadOutcome::default();
+    outcome.skills.push(SkillMetadata {
+        name: "content-tools".to_string(),
+        description: "Prepare visual content.".to_string(),
+        short_description: None,
+        interface: Some(SkillInterface {
+            display_name: Some("Visual Assets".to_string()),
+            short_description: Some("Create illustrated assets.".to_string()),
+            icon_small: None,
+            icon_large: None,
+            brand_color: None,
+            default_prompt: Some("Create an animated pet spritesheet.".to_string()),
+        }),
+        dependencies: None,
+        policy: None,
+        path_to_skills_md: path,
+        scope: SkillScope::User,
+        plugin_id: None,
+        remote_plugin_id: None,
+    });
+    let snapshot = HostSkillsSnapshot::new(std::sync::Arc::new(outcome));
+    let catalog = SkillCatalog {
+        entries: vec![SkillCatalogEntry::new(
+            SkillPackageId(resource.clone()),
+            SkillAuthority::new(SkillSourceKind::Host, "host"),
+            "content-tools",
+            "Prepare visual content.",
+            SkillResourceId::new(resource),
+        )],
+        ..Default::default()
+    };
+    let selector = CharacterRoutingCardSkillSelector::new(&catalog, Some(&snapshot));
+    let documents = [document(
+        /*id*/ 0,
+        "content-tools",
+        "Prepare visual content.",
+    )];
+
+    let selection = selector.select(
+        "make an animated pet spritesheet",
+        &documents,
+        /*limit*/ 20,
+    );
+
+    assert_eq!(vec![0], selection.candidate_ids);
+}
+
+#[test]
+fn connector_dependencies_make_generic_skills_discoverable() {
+    let dependencies = SkillDependencies {
+        tools: vec![SkillToolDependency {
+            r#type: "app".to_string(),
+            value: "slack".to_string(),
+            description: Some("Post messages to team channels.".to_string()),
+            transport: None,
+            command: None,
+            url: None,
+        }],
+    };
+    let documents = [
+        SkillSelectionDocument {
+            dependencies: Some(&dependencies),
+            ..document(
+                /*id*/ 3,
+                "team-communication",
+                "Share updates with the team.",
+            )
+        },
+        document(/*id*/ 4, "documents", "Write project updates."),
+    ];
+    let selector = CharacterRoutingCardSkillSelector::new(
+        &SkillCatalog::default(),
+        /*host_snapshot*/ None,
+    );
+
+    let selection = selector.select("post this update in Slack", &documents, /*limit*/ 20);
+
+    assert_eq!(Some(3), selection.candidate_ids.first().copied());
+}
+
+#[test]
+fn strong_baseline_match_is_preserved_alongside_routing_match() {
+    let documents = [
+        document(/*id*/ 1, "presentation-tools", "Build meeting decks."),
+        document(/*id*/ 2, "content-tools", "Prepare visual content."),
+    ];
+    let selector = CharacterRoutingCardSkillSelector {
+        routing_fields: HashMap::from([(
+            2,
+            "Create a presentation with PowerPoint slides".to_string(),
+        )]),
+    };
+
+    let selection = selector.select(
+        "create a PowerPoint presentation",
+        &documents,
+        /*limit*/ 2,
+    );
+
+    assert_eq!(vec![2, 1], selection.candidate_ids);
+}
+
+#[test]
+fn long_routing_metadata_preserves_original_short_description_matches() {
+    let metadata = (0_u64..60)
+        .map(|index| format!("{:016x}", index.wrapping_mul(/*rhs*/ 0x9e37_79b9_7f4a_7c15)))
+        .collect::<Vec<_>>()
+        .join(" ");
+    let short_description = "zebra xylophone quasar";
+    let documents = [SkillSelectionDocument {
+        short_description: Some(short_description),
+        ..document(/*id*/ 0, "content-tools", "Prepare visual content.")
+    }];
+    let selector = CharacterRoutingCardSkillSelector {
+        routing_fields: HashMap::from([(/*id*/ 0, metadata)]),
+    };
+
+    let selection = selector.select(short_description, &documents, /*limit*/ 20);
+
+    assert_eq!(vec![0], selection.candidate_ids);
+}
+
+fn document<'a>(id: usize, name: &'a str, description: &'a str) -> SkillSelectionDocument<'a> {
+    SkillSelectionDocument {
+        id,
+        name,
+        short_description: None,
+        description,
+        dependencies: None,
+    }
+}

codex-rs/ext/skills/src/extension.rs

@@ -417,10 +417,14 @@ where
                 if host_catalog_in_world_state && host_snapshot.is_some() {
                     shadow_catalog.extend(self.providers.list_host_for_turn(host_query).await);
                 }
-                Some(
-                    self.shadow_selection
-                        .run(&input.user_input, &shadow_catalog),
-                )
+                let shadow_selected_entries =
+                    collect_explicit_skill_mentions(&input.user_input, &shadow_catalog);
+                Some(self.shadow_selection.run(
+                    &input.user_input,
+                    &shadow_catalog,
+                    &shadow_selected_entries,
+                    host_snapshot.as_deref(),
+                ))
             } else {
                 None
             };

codex-rs/ext/skills/src/shadow_selection_experiment.rs

@@ -6,12 +6,15 @@ use std::sync::PoisonError;
 use std::time::Duration;
 use std::time::Instant;
 
+use codex_core_skills::HostSkillsSnapshot;
 use codex_otel::MetricsClient;
 use codex_protocol::user_input::UserInput;
 
 use crate::catalog::SkillCatalog;
+use crate::catalog::SkillCatalogEntry;
 use crate::catalog::SkillSourceKind;
 use crate::dynamic_skill_selector::CharacterNgramSkillSelector;
+use crate::dynamic_skill_selector::CharacterRoutingCardSkillSelector;
 use crate::dynamic_skill_selector::CheapSkillSelection;
 use crate::dynamic_skill_selector::CheapSkillSelector;
 use crate::dynamic_skill_selector::FieldedBm25SkillSelector;
@@ -56,9 +59,15 @@ impl ShadowSelectionExperiment {
         &self,
         inputs: &[UserInput],
         catalog: &SkillCatalog,
+        explicitly_selected: &[SkillCatalogEntry],
+        host_snapshot: Option<&HostSkillsSnapshot>,
     ) -> ShadowSelectionTurnState {
         let query = build_shadow_query(inputs);
         let query_script = query_script_tag(&query.text);
+        let explicitly_selected_skill_resources = explicitly_selected
+            .iter()
+            .map(|entry| normalize_skill_resource(entry.main_prompt.as_str()))
+            .collect::<HashSet<_>>();
         let documents = catalog
             .entries
             .iter()
@@ -71,6 +80,8 @@ impl ShadowSelectionExperiment {
                         &entry.authority.kind,
                         SkillSourceKind::Host | SkillSourceKind::Orchestrator
                     )
+                    && !explicitly_selected_skill_resources
+                        .contains(&normalize_skill_resource(entry.main_prompt.as_str()))
             })
             .map(|(id, entry)| SkillSelectionDocument {
                 id,
@@ -84,9 +95,23 @@ impl ShadowSelectionExperiment {
             .iter()
             .map(|document| document.id)
             .collect::<HashSet<_>>();
-        let mut ranked_selections = Vec::with_capacity(self.selectors.len());
+        let eligible_skill_resources = documents
+            .iter()
+            .map(|document| {
+                normalize_skill_resource(catalog.entries[document.id].main_prompt.as_str())
+            })
+            .collect::<HashSet<_>>();
+        let routing_selector = CharacterRoutingCardSkillSelector::new(catalog, host_snapshot);
+        let mut ranked_selections = Vec::with_capacity(self.selectors.len() + 1);
 
-        for selector in &self.selectors {
+        for selector in self
+            .selectors
+            .iter()
+            .map(std::convert::AsRef::as_ref)
+            .chain(std::iter::once(
+                &routing_selector as &dyn CheapSkillSelector,
+            ))
+        {
             let start = Instant::now();
             let selection =
                 selector.select(&query.text, &documents, /*limit*/ MAX_SHADOW_RESULTS);
@@ -123,12 +148,16 @@ impl ShadowSelectionExperiment {
         ShadowSelectionTurnState {
             ranked_selections,
             query_script,
+            eligible_skill_resources,
             seen_skill_resources: Mutex::new(HashSet::new()),
         }
     }
 
     pub(crate) fn record_invocation(&self, state: &ShadowSelectionTurnState, skill_resource: &str) {
         let skill_resource = normalize_skill_resource(skill_resource);
+        if !state.eligible_skill_resources.contains(&skill_resource) {
+            return;
+        }
         if !state
             .seen_skill_resources
             .lock()
@@ -208,6 +237,7 @@ impl ShadowSelectionExperiment {
 pub(crate) struct ShadowSelectionTurnState {
     ranked_selections: Vec<RankedSelection>,
     query_script: &'static str,
+    eligible_skill_resources: HashSet<String>,
     seen_skill_resources: Mutex<HashSet<String>>,
 }