1- // 70% threshold: Raised from 60% to reduce false positives from short queries.
2- // At 60%, "billing" matched "installing" and "pricing" matched "writing pr descriptions" .
3- // 70% still captures singular/plural ("agent"→"agents" = 80%, "repository"→"repositories" = 73%)
4- // while filtering the worst noise ( both of those false positives were at 67%) .
1+ // The 70% threshold keeps singular/plural matches like "agent" to "agents" at 80%
2+ // and "repository" to "repositories" at 73% .
3+ // It rejects noisy matches like "billing" to "installing" and "pricing" to
4+ // "writing pr descriptions", both 67%.
55const BIGRAM_COVERAGE_THRESHOLD = 0.7
66
7- // Short search terms produce very few bigrams, making spurious matches likely.
8- // Require exact substring match for terms with 4 or fewer non-space characters .
7+ // Terms with 4 or fewer non-space characters need exact substring matches
8+ // because they produce too few bigrams .
99const SHORT_TERM_MAX_LENGTH = 4
1010
1111const bigramCache = new Map < string , Set < string > > ( )
1212
13- // Extract character bigrams from a string, e.g. "agent" -> [ "ag", "ge", "en", "nt"] .
13+ // Bigrams are adjacent character pairs, so "agent" becomes "ag", "ge", "en", and "nt".
1414const getBigrams = ( str : string ) : Set < string > => {
1515 const key = str . toLowerCase ( )
1616 if ( bigramCache . has ( key ) ) {
@@ -27,8 +27,8 @@ const getBigrams = (str: string): Set<string> => {
2727 return bigrams
2828}
2929
30- // Coverage: what percentage of search bigrams are found in text
31- // Better for matching short queries against long text
30+ // Bigram coverage measures how many search bigrams appear in text.
31+ // This works better than Jaccard for short queries against longer text.
3232export const bigramCoverage = ( text : string , search : string ) : number => {
3333 const textBigrams = getBigrams ( text )
3434 const searchBigrams = getBigrams ( search )
@@ -39,19 +39,15 @@ export const bigramCoverage = (text: string, search: string): number => {
3939 return found / searchBigrams . size
4040}
4141
42- // Returns a match score: 1 for exact match, 0-1 for bigram coverage, -1 for no match
42+ // Returns 1 for a substring match, bigram coverage at or above the threshold, or -1 otherwise.
4343export const fuzzyMatchScore = ( text : string , searchTerm : string ) : number => {
4444 const lowerText = text . toLowerCase ( )
4545 const lowerSearch = searchTerm . toLowerCase ( )
4646
4747 if ( lowerText . includes ( lowerSearch ) ) return 1
4848
49- // Short search terms (e.g., "mcp", "pr", "test") produce too few bigrams
50- // for reliable fuzzy matching, so require exact substring only.
5149 if ( lowerSearch . replace ( / \s + / g, '' ) . length <= SHORT_TERM_MAX_LENGTH ) return - 1
5250
53- // Bigram coverage works better than Jaccard when the text is much longer than
54- // the search term.
5551 const score = bigramCoverage ( text , searchTerm )
5652 return score >= BIGRAM_COVERAGE_THRESHOLD ? score : - 1
5753}
@@ -60,9 +56,8 @@ export const fuzzyMatch = (text: string, searchTerm: string): boolean => {
6056 return fuzzyMatchScore ( text , searchTerm ) >= 0
6157}
6258
63- // Strip stop words from a string, preserving other words.
64- // On product-specific landing pages (e.g. /copilot), the product name appears
65- // in nearly every article, drowning out the actual query.
59+ // Product-specific landing pages, such as /copilot, repeat the product name in nearly
60+ // every article, so stop words keep the product name from drowning out the query.
6661export const stripStopWords = ( text : string , stopWords : string [ ] ) : string =>
6762 text
6863 . split ( / \s + / )
0 commit comments