From edc27cba38de0d234341882d289c33019e646053 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Fri, 16 Oct 2026 14:15:18 +0200 Subject: [PATCH] fix(search): match language tags against locales as language ranges - Add matchLanguage to @lde/search/adapter: BCP-47 range matching, exact tag first, then truncations, extensions and sibling regions; und matches only und - Index a literal tagged nl-NL under a declared nl, exact tag first - Sort and boost a request in nl-NL on the nl keys instead of the first locale - Order a GraphQL language list by range, so nl-NL lists the nl value first - Document the matching in the Locales reference --- docs/reference/search.md | 8 ++- packages/search-api-graphql/src/language.ts | 14 ++-- .../search-api-graphql/test/language.test.ts | 32 +++++++++ .../search-typesense/src/query-compiler.ts | 13 ++-- .../test/query-compiler.test.ts | 22 ++++++ packages/search-typesense/vite.config.ts | 2 +- packages/search/src/adapter.ts | 1 + packages/search/src/language.ts | 67 +++++++++++++++++++ packages/search/src/project.ts | 38 +++++++++-- packages/search/test/language.test.ts | 61 +++++++++++++++++ packages/search/test/project.test.ts | 31 +++++++++ packages/search/vite.config.ts | 2 +- 12 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 packages/search-api-graphql/test/language.test.ts create mode 100644 packages/search/src/language.ts create mode 100644 packages/search/test/language.test.ts diff --git a/docs/reference/search.md b/docs/reference/search.md index 2fe3bc0f..68eba42a 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -1042,7 +1042,13 @@ A field with `searchable` but no `output` is **search-only** – folded and stem for retrieval but never rendered (e.g. a creator searched here but shown via a separate label). **Only listed locales are indexed** (searched and sorted); a literal whose language tag is not in `locales` is still **displayed** but not -matched or sorted on. Display fields are **omitted, never empty**, when a document +matched or sorted on. A tag is matched against `locales` as a BCP-47 language +range, not as an exact string: a declared `nl` indexes literals tagged `nl`, +`nl-NL` and `nl-BE`, the exact tag first. A request in `nl-NL` is boosted and +sorted on the `nl` keys, and the GraphQL surface lists a document’s `nl` value +first. A sibling region (`nl-BE` for `nl-NL`) matches only after the exact tag +and the more general one. The exact match still wins where both are declared +(`en` beside `en-GB`), and `und` matches nothing but untagged literals. Display fields are **omitted, never empty**, when a document lacks that language, and the per-locale search fields likewise, so declare them optional in the engine schema and sort with `missing_values: last`. A sort field is present whenever the field holds any value in a declared locale (the fallback diff --git a/packages/search-api-graphql/src/language.ts b/packages/search-api-graphql/src/language.ts index 96826f65..05667032 100644 --- a/packages/search-api-graphql/src/language.ts +++ b/packages/search-api-graphql/src/language.ts @@ -1,4 +1,5 @@ import type { LocalizedValue } from '@lde/search'; +import { matchLanguage } from '@lde/search/adapter'; /** One entry of the surface’s best-first `[LanguageString!]!`. `language` is null * for untagged (`und`) values; `[0]` is the value to display and `[0].language` @@ -15,12 +16,17 @@ export type LanguageOrder = ( ) => readonly string[]; /** - * Default ordering: requested languages first (in request order), then the - * remaining tagged languages, then untagged (`und`) last — so `[0]` is always the - * best available value. + * Default ordering: requested languages first (in request order, each matched + * as a language range – `nl-NL` finds `nl` – most specific first), then the + * remaining tagged languages, then untagged (`und`) last – so `[0]` is always + * the best available value. */ export const defaultLanguageOrder: LanguageOrder = (available, accept) => { - const requested = accept.filter((language) => available.includes(language)); + const requested = [ + ...new Set( + accept.flatMap((language) => matchLanguage(language, available)), + ), + ]; const rest = available.filter( (language) => language !== 'und' && !requested.includes(language), ); diff --git a/packages/search-api-graphql/test/language.test.ts b/packages/search-api-graphql/test/language.test.ts new file mode 100644 index 00000000..641ea20c --- /dev/null +++ b/packages/search-api-graphql/test/language.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; +import { defaultLanguageOrder } from '../src/language.js'; + +describe('defaultLanguageOrder', () => { + it('puts requested languages first, then the rest, then und', () => { + expect(defaultLanguageOrder(['und', 'nl', 'en', 'fr'], ['en'])).toEqual([ + 'en', + 'nl', + 'fr', + 'und', + ]); + }); + + it('finds the primary language for a regional request', () => { + // A browser’s `nl-NL` must surface the Dutch value, not whatever language + // the document happens to list first. + expect(defaultLanguageOrder(['en', 'nl'], ['nl-NL'])).toEqual(['nl', 'en']); + }); + + it('finds regional values for a primary-language request', () => { + expect(defaultLanguageOrder(['en', 'nl-BE'], ['nl'])).toEqual([ + 'nl-BE', + 'en', + ]); + }); + + it('ranks the exact tag before a general one, without duplicates', () => { + expect( + defaultLanguageOrder(['en', 'en-GB', 'nl'], ['en-GB', 'en', 'nl']), + ).toEqual(['en-GB', 'en', 'nl']); + }); +}); diff --git a/packages/search-typesense/src/query-compiler.ts b/packages/search-typesense/src/query-compiler.ts index cdc0bc2f..83f8b738 100644 --- a/packages/search-typesense/src/query-compiler.ts +++ b/packages/search-typesense/src/query-compiler.ts @@ -24,6 +24,7 @@ import { isWelded, joinGraph, localLookupTypeOf, + matchLanguage, pageForOffset, physicalFields, nestedReferenceType, @@ -240,13 +241,14 @@ function collectSearchable( const baseWeight = field.searchable.weight; if (field.kind === 'text') { const locales = field.locales; + const activeLocale = matchLanguage(locale, locales)[0]; search.forEach((name, index) => { names.push(qualify(prefix, name)); // The active locale keeps full weight; `und` is language-neutral, so // it is never demoted (an untagged-only field would otherwise always // rank below its declared weight). weights.push( - locales[index] === locale || locales[index] === 'und' + locales[index] === activeLocale || locales[index] === 'und' ? baseWeight : Math.max(1, baseWeight - 1), ); @@ -645,9 +647,12 @@ function compileSort( } const field = fieldNamed(searchType, sort.field); if (field !== undefined && field.kind === 'text') { - // Sort on the active locale's key, falling back to the first declared - // locale (e.g. an und-only field sorted by a user in any language). - const index = field.locales.indexOf(locale); + // Sort on the key of the declared locale the active one best matches as a + // language range (`nl-NL` sorts on `nl`), falling back to the first + // declared locale (e.g. an und-only field sorted by a user in any language). + const activeLocale = matchLanguage(locale, field.locales)[0]; + const index = + activeLocale === undefined ? -1 : field.locales.indexOf(activeLocale); const sortName = physicalFields(field).sort[index === -1 ? 0 : index]; if (sortName !== undefined) { return `${sortName}:${sort.direction}`; diff --git a/packages/search-typesense/test/query-compiler.test.ts b/packages/search-typesense/test/query-compiler.test.ts index 123f6236..3f99bef0 100644 --- a/packages/search-typesense/test/query-compiler.test.ts +++ b/packages/search-typesense/test/query-compiler.test.ts @@ -84,6 +84,28 @@ describe('buildSearchParams', () => { ).toBe('4,5,1'); }); + it('boosts the declared locale a regional tag falls under', () => { + expect( + buildSearchParams({ ...base, text: 'Kaart', locale: 'en-GB' }, schema) + .query_by_weights, + ).toBe('4,5,1'); + }); + + it('sorts a regional tag on its declared locale’s key', () => { + // locales is ['nl', 'en']: without range matching `en-GB` would fall back + // to the first locale and sort English readers by the Dutch title. + expect( + buildSearchParams( + { + ...base, + locale: 'en-GB', + orderBy: [{ field: 'title', direction: 'asc' }], + }, + schema, + ).sort_by, + ).toBe('title_sort_en:asc'); + }); + it('maps offset/limit to numbered pages', () => { expect( buildSearchParams({ ...base, offset: 40, limit: 20 }, schema).page, diff --git a/packages/search-typesense/vite.config.ts b/packages/search-typesense/vite.config.ts index 15ebf0ea..69821279 100644 --- a/packages/search-typesense/vite.config.ts +++ b/packages/search-typesense/vite.config.ts @@ -28,7 +28,7 @@ export default mergeConfig( // projection naming what no lookup reaches are unreachable through // the port, since `assertValidQuery` rejects such a query first. // They hold for a direct caller, and are exercised as one. - branches: 95.89, + branches: 95.9, statements: 99.48, }, }, diff --git a/packages/search/src/adapter.ts b/packages/search/src/adapter.ts index 3437e99f..94f2db28 100644 --- a/packages/search/src/adapter.ts +++ b/packages/search/src/adapter.ts @@ -47,6 +47,7 @@ export { } from './schema.js'; export type { PhysicalFields } from './schema.js'; export { physicalNameTokens } from './physical-name.js'; +export { matchLanguage } from './language.js'; export { joinGraph, MAX_JOIN_DEPTH } from './join-graph.js'; export type { JoinGraph } from './join-graph.js'; export { diff --git a/packages/search/src/language.ts b/packages/search/src/language.ts new file mode 100644 index 00000000..24d54987 --- /dev/null +++ b/packages/search/src/language.ts @@ -0,0 +1,67 @@ +/** + * The entries of `candidates` that language tag `tag` matches, best first, as + * the entries themselves (so a caller can index back into `candidates`). + * + * Tags are compared per BCP-47 subtag and case-insensitively, as language + * ranges (RFC 4647). A candidate matches when it shares the tag’s primary + * language, ranked: + * + * 1. the exact tag (`nl-NL` for `nl-NL`); + * 2. the tag’s truncations, most specific first – RFC 4647 lookup, so a + * request for `nl-NL` finds a declared `nl`; + * 3. the tags it is a prefix of, in `candidates` order – RFC 4647 basic + * filtering, so a range `nl` finds `nl-NL` and `nl-BE`; + * 4. the remaining tags of the same language, longest shared prefix first, so + * a request for `nl-NL` still finds `nl-BE` rather than another language. + * + * The exact match therefore still wins where a general and a specific tag sit + * side by side (`en` and `en-GB`). `und`, the untagged bucket, matches nothing + * but `und`. + */ +export function matchLanguage( + tag: string, + candidates: readonly string[], +): string[] { + const subtags = tag.toLowerCase().split('-'); + const ranked: { candidate: string; kind: number; closeness: number }[] = []; + for (const candidate of candidates) { + const candidateSubtags = candidate.toLowerCase().split('-'); + const shared = sharedPrefixLength(subtags, candidateSubtags); + const exact = + shared === subtags.length && shared === candidateSubtags.length; + if (shared === 0 || (subtags[0] === 'und' && !exact)) { + continue; + } + const kind = exact + ? 0 + : shared === candidateSubtags.length + ? 1 + : shared === subtags.length + ? 2 + : 3; + // Extensions keep candidate order; truncations and siblings rank by how + // much of the tag they share. + ranked.push({ candidate, kind, closeness: kind === 2 ? 0 : shared }); + } + return ranked + .sort( + (left, right) => + left.kind - right.kind || right.closeness - left.closeness, + ) + .map(({ candidate }) => candidate); +} + +function sharedPrefixLength( + left: readonly string[], + right: readonly string[], +): number { + let length = 0; + while ( + length < left.length && + length < right.length && + left[length] === right[length] + ) { + length++; + } + return length; +} diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 2a71b12c..b962068e 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -1,5 +1,6 @@ import type { Quad } from '@rdfjs/types'; import { fold } from '@lde/text-normalization'; +import { matchLanguage } from './language.js'; import { buildSubjectIndex, frameSubjects, @@ -562,11 +563,38 @@ function applyText( // here it simply indexes nothing. if (field.searchable !== undefined || field.sortable === true) { const names = physicalFields(field); - const valuesPerLocale = field.locales.map((locale) => - values - .filter((value) => value.lang === locale) - .map((value) => value.value), - ); + // A literal lands under the declared locale its tag best matches as a + // language range, so `nl-NL` titles are keyed under a declared `nl`; within + // a locale, the exact tag comes first. + const valuesPerLanguage = new Map(); + for (const { lang, value } of values) { + const languageValues = valuesPerLanguage.get(lang); + if (languageValues === undefined) { + valuesPerLanguage.set(lang, [value]); + } else { + languageValues.push(value); + } + } + const languagesPerLocale = new Map(); + for (const entry of valuesPerLanguage) { + const locale = matchLanguage(entry[0], field.locales)[0]; + if (locale !== undefined) { + languagesPerLocale.set(locale, [ + ...(languagesPerLocale.get(locale) ?? []), + entry, + ]); + } + } + const valuesPerLocale = field.locales.map((locale) => { + const entries = languagesPerLocale.get(locale) ?? []; + const ranked = matchLanguage( + locale, + entries.map(([language]) => language), + ); + return entries + .sort(([left], [right]) => ranked.indexOf(left) - ranked.indexOf(right)) + .flatMap(([, languageValues]) => languageValues); + }); // What every locale’s sort key falls back to: the document’s first value // in `locales` order, which is the declaration’s own statement of which // language stands in for the others. diff --git a/packages/search/test/language.test.ts b/packages/search/test/language.test.ts new file mode 100644 index 00000000..8ba6e991 --- /dev/null +++ b/packages/search/test/language.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest'; +import { matchLanguage } from '../src/language.js'; + +describe('matchLanguage', () => { + it('matches an exact tag', () => { + expect(matchLanguage('nl', ['en', 'nl'])).toEqual(['nl']); + }); + + it('finds a declared primary subtag for a regional tag', () => { + expect(matchLanguage('nl-NL', ['en', 'nl'])).toEqual(['nl']); + }); + + it('finds regional tags for a primary subtag, in candidate order', () => { + expect(matchLanguage('nl', ['nl-NL', 'en', 'nl-BE'])).toEqual([ + 'nl-NL', + 'nl-BE', + ]); + }); + + it('ranks the exact tag first, then truncations, then extensions', () => { + expect( + matchLanguage('zh-Hant', ['zh-Hant-TW', 'zh', 'zh-Hant', 'en']), + ).toEqual(['zh-Hant', 'zh', 'zh-Hant-TW']); + expect(matchLanguage('zh-Hant-TW', ['zh', 'zh-Hant'])).toEqual([ + 'zh-Hant', + 'zh', + ]); + }); + + it('lets the exact tag win where general and specific sit side by side', () => { + expect(matchLanguage('en', ['en-GB', 'en'])).toEqual(['en', 'en-GB']); + expect(matchLanguage('en-GB', ['en', 'en-GB'])).toEqual(['en-GB', 'en']); + }); + + it('falls back to a sibling region after every closer match', () => { + expect(matchLanguage('nl-NL', ['nl-BE', 'en', 'nl'])).toEqual([ + 'nl', + 'nl-BE', + ]); + expect( + matchLanguage('zh-Hant-TW', ['zh-Hans', 'zh-Hant-HK', 'zh-TW']), + ).toEqual(['zh-Hant-HK', 'zh-Hans', 'zh-TW']); + }); + + it('does not match another language', () => { + expect(matchLanguage('nl-NL', ['en', 'fr-BE'])).toEqual([]); + }); + + it('compares case-insensitively and returns the candidate as given', () => { + expect(matchLanguage('pt-br', ['PT', 'pt-BR'])).toEqual(['pt-BR', 'PT']); + }); + + it('compares whole subtags, not string prefixes', () => { + expect(matchLanguage('e', ['en'])).toEqual([]); + }); + + it('matches und only to und', () => { + expect(matchLanguage('und', ['nl', 'und'])).toEqual(['und']); + expect(matchLanguage('nl', ['und'])).toEqual([]); + }); +}); diff --git a/packages/search/test/project.test.ts b/packages/search/test/project.test.ts index 8b6a09d9..391afd91 100644 --- a/packages/search/test/project.test.ts +++ b/packages/search/test/project.test.ts @@ -388,6 +388,37 @@ describe('projectDocument', () => { expect(document.title_sort_nl).toBeUndefined(); }); + it('indexes a regional tag under its declared primary language', () => { + const document = projectDocument( + { + '@id': 'https://ex/d/6a', + [dsKey('title')]: { '@language': 'nl-NL', '@value': 'Kaart' }, + }, + { name: 'Dataset', class: DATASET, fields }, + ); + // locales is ['nl', 'en']: `nl-NL` is Dutch, so it is searched and sorted + // under `nl` – and displayed under the tag the publisher gave it. + expect(document['title_nl-NL']).toBe('Kaart'); + expect(document.title_search_nl).toBe('kaart'); + expect(document.title_sort_nl).toBe('kaart'); + expect(document.title_search_en).toBeUndefined(); + }); + + it('puts the exact tag first when a regional one shares its locale', () => { + const document = projectDocument( + { + '@id': 'https://ex/d/6c', + [dsKey('title')]: [ + { '@language': 'nl-BE', '@value': 'Plan' }, + { '@language': 'nl', '@value': 'Kaart' }, + ], + }, + { name: 'Dataset', class: DATASET, fields }, + ); + expect(document.title_sort_nl).toBe('kaart'); + expect(document.title_search_nl).toBe('kaart plan'); + }); + it('normalises an underscore-style language tag to its BCP-47 shape', () => { const document = projectDocument( { diff --git a/packages/search/vite.config.ts b/packages/search/vite.config.ts index 92874588..946fff9b 100644 --- a/packages/search/vite.config.ts +++ b/packages/search/vite.config.ts @@ -12,7 +12,7 @@ export default mergeConfig( thresholds: { functions: 100, lines: 100, - branches: 99.72, + branches: 99.73, statements: 100, }, },