Cherry-pick: fix(sdk): skill search keeps words in every script and leaves out stop words (#1065, #1061)

apps/e2e/demo-e2e-bundle-size/e2e/browser-bundle-size.e2e.spec.ts

@@ -21,8 +21,8 @@ import {
 } from './helpers/browser-bundle';
 
 const BUDGETS: Array<{ entry: string; label: string; maxBytes: number }> = [
-  { entry: 'create-tool.ts', label: 'create() + one tool', maxBytes: 1_287_930 },
-  { entry: 'create-webmcp-tool.ts', label: 'create() + WebMcpPlugin + one tool', maxBytes: 1_295_119 },
+  { entry: 'create-tool.ts', label: 'create() + one tool', maxBytes: 1_287_985 },
+  { entry: 'create-webmcp-tool.ts', label: 'create() + WebMcpPlugin + one tool', maxBytes: 1_295_174 },
 ];
 
 const VITE_INITIAL_CHUNK_MAX_BYTES = 762_830;

libs/cli/package.json

@@ -45,7 +45,7 @@
     "esbuild": "^0.27.3",
     "jsonc-parser": "^3.2.0",
     "tslib": "^2.3.0",
-    "vectoriadb": "^2.3.3",
+    "vectoriadb": "^2.3.4",
     "yauzl": "^3.2.0",
     "yazl": "^3.3.1"
   },

libs/sdk/package.json

@@ -180,7 +180,7 @@
     "js-yaml": "^4.1.1",
     "reflect-metadata": "^0.2.2",
     "semver": "^7.6.0",
-    "vectoriadb": "^2.3.3"
+    "vectoriadb": "^2.3.4"
   },
   "devDependencies": {
     "typescript": "^5.9.3"

libs/sdk/src/skill/__tests__/memory-skill.provider.tokenizer-cache.spec.ts

@@ -0,0 +1,62 @@
+/**
+ * The index snapshot cache key names the tokenizer, so a snapshot cached before vectoriadb 2.3.4, whose terms are
+ * ASCII-only, is not restored and re-indexed on every cold start: the first search rebuilds and saves a snapshot
+ * under the new key, and the next cold start restores it.
+ */
+import { TFIDFVectoria } from 'vectoriadb';
+
+import { sha256Hex } from '@frontmcp/utils';
+
+import type { SkillContent } from '../../common/interfaces';
+import { MemorySkillProvider } from '../providers/memory-skill.provider';
+import type { SkillIndexCache } from '../skill-index-cache.interface';
+import { skillSearchText } from '../skill-search-text';
+
+const skill: SkillContent = {
+  id: 'katalog-he',
+  name: 'katalog-he',
+  description: 'מציאת כלבים, חתולים וציפורים',
+  instructions: '',
+  tools: [],
+};
+
+function mapCache(): SkillIndexCache & { entries: Map<string, unknown> } {
+  const entries = new Map<string, unknown>();
+  return {
+    entries,
+    get: async (key) => entries.get(key),
+    set: async (key, value) => {
+      entries.set(key, value);
+    },
+  };
+}
+
+async function providerWith(cache: SkillIndexCache): Promise<MemorySkillProvider> {
+  const provider = new MemorySkillProvider({ indexCache: cache });
+  await provider.initialize();
+  await provider.add(skill);
+  return provider;
+}
+
+describe('the skill index snapshot cache and the tokenizer (#1065)', () => {
+  it('saves a snapshot for the new tokenizer once, and restores it on the next cold start', async () => {
+    const cache = mapCache();
+    const legacyKey = sha256Hex(`frontmcp-skill-index:v1|cosine|1|${skill.id}\u0000${skillSearchText(skill)}`);
+    cache.entries.set(legacyKey, { legacy: true });
+
+    const first = await providerWith(cache);
+    expect((await first.search('חתולים')).map((r) => r.metadata.id)).toEqual(['katalog-he']);
+
+    const savedKeys = [...cache.entries.keys()].filter((key) => key !== legacyKey);
+    expect(savedKeys).toHaveLength(1);
+
+    const reindex = jest.spyOn(TFIDFVectoria.prototype, 'reindex');
+    try {
+      const second = await providerWith(cache);
+      expect((await second.search('חתולים')).map((r) => r.metadata.id)).toEqual(['katalog-he']);
+      expect(reindex).not.toHaveBeenCalled();
+    } finally {
+      reindex.mockRestore();
+    }
+  });
+});

libs/sdk/src/skill/__tests__/skill-search-text.spec.ts

@@ -1,3 +1,5 @@
+import { defaultTokenizer } from 'vectoriadb';
+
 import { type SkillContent } from '../../common/interfaces';
 import {
   identifierWords,
@@ -7,6 +9,7 @@ import {
   queryTerms,
   queryTermShare,
   skillSearchText,
+  skillTokenizer,
 } from '../skill-search-text';
 
 const skill = (overrides: Partial<SkillContent> = {}): SkillContent => ({
@@ -105,3 +108,22 @@ describe('negativeQueryShare', () => {
     expect(negativeQueryShare(undefined, 'count the pets in stock')).toBe(0);
   });
 });
+
+describe('skillTokenizer (#1061, #1065)', () => {
+  const tokenize = skillTokenizer(defaultTokenizer);
+
+  it("leaves out stop words, and keeps vectoriadb's terms in every script", () => {
+    expect(tokenize('Show me the cats')).toEqual(['show', 'cats']);
+    expect(tokenize('מציאת חתולים')).toEqual(['מציאת', 'חתולים']);
+    expect(tokenize('查找狗')).toEqual(['查', '查找', '找', '找狗', '狗']);
+    expect(tokenize('Vögel')).toEqual(['vogel']);
+  });
+
+  it('matches the text as the index sees it, a term within a word included', () => {
+    expect(queryTermShare(queryTerms('Vogel', tokenize), 'Hunde, Katzen und Vögel finden.', tokenize)).toBe(1);
+    expect(queryTermShare(queryTerms('refund', tokenize), 'Process refunds', tokenize)).toBe(1);
+    expect(queryTermShare(queryTerms('猫', tokenize), '查找狗、猫和鸟。', tokenize)).toBe(1);
+    expect(queryTermShare(queryTerms('show me the cats', tokenize), 'Find dogs and cats', tokenize)).toBe(0.5);
+    expect(queryTermShare(queryTerms('the', tokenize), 'the store', tokenize)).toBe(0);
+  });
+});

libs/sdk/src/skill/providers/memory-skill.provider.ts

@@ -8,7 +8,15 @@ import { type SkillContent } from '../../common/interfaces';
 import { type SkillMetadata, type SkillVisibility } from '../../common/metadata';
 import { importWithRequireFallback } from '../../utils/dynamic-import.utils';
 import { type SkillIndexCache, type SkillIndexScoring } from '../skill-index-cache.interface';
-import { negativeQueryShare, queryTerms, queryTermShare, skillSearchText } from '../skill-search-text';
+import {
+  negativeQueryShare,
+  queryTerms,
+  queryTermShare,
+  SKILL_TOKENIZER_ID,
+  skillSearchText,
+  skillTokenizer,
+  type SkillTokenizer,
+} from '../skill-search-text';
 import {
   type MutableSkillStorageProvider,
   type SkillListOptions,
@@ -116,6 +124,15 @@ export class MemorySkillProvider implements MutableSkillStorageProvider {
   /** Optional snapshot cache for fast cold-start (e.g. KV-backed on the edge). */
   private indexCache?: SkillIndexCache;
 
+  /**
+   * The index's tokenizer, set when the index is created: vectoriadb's (2.3.4 and later), without stop words. The
+   * text fallback matches with it too. Unset on an older vectoriadb, which tokenizes ASCII only.
+   */
+  private tokenizer?: SkillTokenizer;
+
+  /** The id of {@link tokenizer}, part of the snapshot cache key; unset with an older vectoriadb's tokenizer. */
+  private tokenizerId?: string;
+
   /**
    * The index is rebuilt from `skills` when it is stale: adds/removes bump
    * `skillsVersion`, and the FIRST search after a mutation builds (or restores)
@@ -135,19 +152,38 @@ export class MemorySkillProvider implements MutableSkillStorageProvider {
 
   private async createVectorDB(): Promise<TFIDFVectoria<SkillDocumentMetadata>> {
     // Loaded with the first search, so a server that never searches skills never loads it.
-    const { TFIDFVectoria } = await importWithRequireFallback(
+    const vectoriadb = await importWithRequireFallback(
       () => import('vectoriadb'),
       () => require('vectoriadb') as typeof import('vectoriadb'),
     );
+    const { TFIDFVectoria } = vectoriadb;
     // `scoring` is read by newer vectoriadb versions; older ones ignore the
     // extra field and use cosine. Built as a typed variable (not an object
     // literal) so passing it to the older constructor type isn't an excess-
     // property error.
-    const dbConfig: { defaultTopK: number; defaultSimilarityThreshold: number; scoring?: SkillIndexScoring } = {
+    const dbConfig: {
+      defaultTopK: number;
+      defaultSimilarityThreshold: number;
+      scoring?: SkillIndexScoring;
+      tokenizer?: SkillTokenizer;
+      tokenizerId?: string;
+    } = {
       defaultTopK: this.defaultTopK,
       defaultSimilarityThreshold: this.defaultMinScore,
     };
     if (this.scoring !== 'cosine') dbConfig.scoring = this.scoring;
+    // vectoriadb 2.3.4 and later export a tokenizer that keeps the letters of every script (#1065); the index uses
+    // it without stop words (#1061). An older vectoriadb has none and ignores these fields.
+    const { defaultTokenizer, DEFAULT_TOKENIZER_ID } = vectoriadb as {
+      defaultTokenizer?: SkillTokenizer;
+      DEFAULT_TOKENIZER_ID?: string;
+    };
+    if (typeof defaultTokenizer === 'function') {
+      this.tokenizer = skillTokenizer(defaultTokenizer);
+      dbConfig.tokenizer = this.tokenizer;
+      dbConfig.tokenizerId = `${SKILL_TOKENIZER_ID}+${DEFAULT_TOKENIZER_ID ?? 'unknown'}`;
+      this.tokenizerId = dbConfig.tokenizerId;
+    }
     return new TFIDFVectoria<SkillDocumentMetadata>(dbConfig);
   }
 
@@ -242,15 +278,16 @@ export class MemorySkillProvider implements MutableSkillStorageProvider {
   }
 
   /**
-   * Stable content hash of the indexed skill set + scoring mode. Changing any
-   * skill's searchable text, the set of skills, or the scoring mode changes the
-   * key, so a stale snapshot is never restored.
+   * Stable content hash of the indexed skill set, scoring mode and tokenizer. Changing any skill's searchable text,
+   * the set of skills, the scoring mode or the tokenizer changes the key, so a stale snapshot is never restored, and
+   * the snapshot built for a new tokenizer is saved under a key the next cold start finds.
    */
   private async computeIndexKey(): Promise<string> {
     const parts = Array.from(this.skills.values())
       .map((s) => `${s.id}\u0000${skillSearchText(s)}`)
       .sort();
-    const canonical = `frontmcp-skill-index:v1|${this.scoring}|${this.skills.size}|${parts.join('
')}`;
+    const tokenizer = this.tokenizerId ?? 'ascii';
+    const canonical = `frontmcp-skill-index:v1|${this.scoring}|${tokenizer}|${this.skills.size}|${parts.join('
')}`;
     return sha256Hex(canonical);
   }
 
@@ -536,7 +573,7 @@ export class MemorySkillProvider implements MutableSkillStorageProvider {
     negativeQuery?: string | string[],
     negativeWeight = 1,
   ): Array<{ id: string; text: string; score: number; metadata: SkillDocumentMetadata }> {
-    const terms = queryTerms(query);
+    const terms = queryTerms(query, this.tokenizer);
 
     if (terms.length === 0) {
       return [];
@@ -555,7 +592,8 @@ export class MemorySkillProvider implements MutableSkillStorageProvider {
       // The anti-query demotes here as it does in the index, so this fallback never brings back a skill the
       // index demoted (#1059).
       const score =
-        queryTermShare(terms, searchableText) - negativeWeight * negativeQueryShare(negativeQuery, searchableText);
+        queryTermShare(terms, searchableText, this.tokenizer) -
+        negativeWeight * negativeQueryShare(negativeQuery, searchableText, this.tokenizer);
 
       if (score > 0) {
         matches.push({

libs/sdk/src/skill/skill-search-text.ts

@@ -192,8 +192,29 @@ export function skillSearchText(skill: SkillContent): string {
   return parts.join(' ');
 }
 
-/** A query's terms for {@link queryTermShare}: its lowercase words that aren't stop words. */
-export function queryTerms(query: string): string[] {
+/** Splits text into the terms the skill index counts. */
+export type SkillTokenizer = (text: string) => string[];
+
+/**
+ * The id the skill index records for its tokenizer in snapshots, joined with the id of the vectoriadb tokenizer it
+ * wraps. Change it whenever {@link skillTokenizer} gives other terms, so cached snapshots are re-indexed.
+ */
+export const SKILL_TOKENIZER_ID = 'frontmcp-skills-1';
+
+/**
+ * The skill index's tokenizer: vectoriadb's, which keeps the letters of every script (#1065), without stop words
+ * (#1061), for the index and the query alike.
+ */
+export function skillTokenizer(tokenize: SkillTokenizer): SkillTokenizer {
+  return (text) => tokenize(text).filter((term) => !SKILL_STOP_WORDS.has(term));
+}
+
+/**
+ * A query's terms for {@link queryTermShare}: its tokenizer's terms when there is one, or else its lowercase words
+ * that aren't stop words.
+ */
+export function queryTerms(query: string, tokenize?: SkillTokenizer): string[] {
+  if (tokenize) return [...new Set(tokenize(query))];
   return query
     .toLowerCase()
     .split(/\s+/)
@@ -202,11 +223,14 @@ export function queryTerms(query: string): string[] {
 
 /**
  * The share of a query's terms that the text contains, from 0 to 1: the score of the keyword matches that back up
- * the index (a skill the index has no terms for, or one registered at runtime that it hasn't indexed yet).
+ * the index (a skill the index has no terms for, or one registered at runtime that it hasn't indexed yet). A term
+ * matches a part of a word, so `refund` matches `refunds`. With a tokenizer the text is matched as the index sees it,
+ * its words folded and split the same way as the query's (`vogel` matches `Vögel`).
  */
-export function queryTermShare(terms: readonly string[], text: string): number {
+export function queryTermShare(terms: readonly string[], text: string, tokenize?: SkillTokenizer): number {
   if (terms.length === 0) return 0;
-  const haystack = text.toLowerCase();
+  // Terms hold no spaces, so a term found in the space-joined terms lies within one of them
+  const haystack = tokenize ? ` ${tokenize(text).join(' ')} ` : text.toLowerCase();
   return terms.filter((term) => haystack.includes(term)).length / terms.length;
 }
 
@@ -215,8 +239,12 @@ export function queryTermShare(terms: readonly string[], text: string): number {
  * The keyword matches subtract `negativeWeight` times this from their score, as the index does with its own
  * similarity, so a skill the anti-query demotes stays demoted.
  */
-export function negativeQueryShare(negativeQuery: string | readonly string[] | undefined, text: string): number {
+export function negativeQueryShare(
+  negativeQuery: string | readonly string[] | undefined,
+  text: string,
+  tokenize?: SkillTokenizer,
+): number {
   if (negativeQuery === undefined) return 0;
   const queries = typeof negativeQuery === 'string' ? [negativeQuery] : negativeQuery;
-  return queries.reduce((max, query) => Math.max(max, queryTermShare(queryTerms(query), text)), 0);
+  return queries.reduce((max, query) => Math.max(max, queryTermShare(queryTerms(query, tokenize), text, tokenize)), 0);
 }

libs/skills/catalog/frontmcp-development/references/create-skill.md

@@ -163,6 +163,8 @@ class ApiStandardsSkill extends SkillContext {}
 > **Search index:** `skills/search` ranks with TF-IDF from `vectoriadb`, a dependency of `@frontmcp/sdk`. Under the `browser` and `worker` export conditions it resolves to its TF-IDF/BM25 entry (no Node built-ins), so search works the same in Node, on edge runtimes and in the browser build. The index is created the first time a search runs, not at server start.
 >
 > A skill is matched by its description (weighted 3x), its tags (2x), its name, tools, parameters and examples, and, for a skill registered at runtime such as a Skilled OpenAPI bundle skill, its actions: each action's id split into words (`getInventory` is "get inventory"), summary and description. Its instructions count at a lower weight: their distinct words, up to 48, once each.
+>
+> Words in any script are terms: Hebrew, Arabic and Cyrillic words whole, Chinese as characters and neighbor pairs, Japanese kana and Thai as neighbor pairs, and accents on Latin and Greek letters folded, so `Vogel` matches `Vögel`. Stop words (`the`, `of`, `me`, …) count neither in a skill's text nor in the query. Up to 1.11.0 only `[A-Za-z0-9_]` made terms, and stop words counted.
 
 ## SkillContext: loadInstructions() and build()
 

libs/skills/catalog/frontmcp-development/references/official-plugins.md

@@ -216,7 +216,7 @@ CodeCallPlugin.init({
 
 ### Power Features
 
-- **TF-IDF Search** -- Term frequency-inverse document frequency scoring indexes tool names, descriptions, and tags. No external embedding service required.
+- **TF-IDF Search** -- Term frequency-inverse document frequency scoring indexes tool names, descriptions, and tags. No external embedding service required. Words in any script are terms (`vectoriadb` 2.3.4 and later): Hebrew, Arabic and Cyrillic words whole, Chinese as characters and pairs, Japanese kana and Thai as pairs, and accents on Latin and Greek letters folded. Up to 1.11.0 only `[A-Za-z0-9_]` made terms.
 - **Synonym Expansion** -- Automatically expands search queries with synonyms (e.g., "delete" also matches "remove", "erase"). Enable via `embedding.synonymExpansion.enabled`.
 - **Pass-by-Reference via Sidecar** -- Large results are stored in a sidecar map and passed by reference between tool calls in AgentScript, avoiding serialization overhead.
 

package.json

@@ -52,7 +52,7 @@
     "mcp-from-openapi": "2.9.1",
     "openapi-types": "^12.1.3",
     "reflect-metadata": "^0.2.2",
-    "vectoriadb": "^2.3.3",
+    "vectoriadb": "^2.3.4",
     "yaml": "^2.9.0",
     "zod": "^4.6.1"
   },

plugins/plugin-codecall/package.json

@@ -56,7 +56,7 @@
     "@enclave-vm/core": "^2.17.0",
     "@frontmcp/protocol": "1.11.0",
     "acorn": "^8.15.0",
-    "vectoriadb": "^2.3.3"
+    "vectoriadb": "^2.3.4"
   },
   "peerDependencies": {
     "@frontmcp/lazy-zod": "1.11.0",