fix(search): match language tags against locales as language ranges

docs/reference/search.md

@@ -1042,7 +1042,13 @@ A field with `searchable` but no `output` is **search-only** – folded and stem
 for retrieval but never rendered (e.g. a creator searched here but shown via a
 separate label). **Only listed locales are indexed** (searched and sorted); a
 literal whose language tag is not in `locales` is still **displayed** but not
-matched or sorted on. Display fields are **omitted, never empty**, when a document
+matched or sorted on. A tag is matched against `locales` as a BCP-47 language
+range, not as an exact string: a declared `nl` indexes literals tagged `nl`,
+`nl-NL` and `nl-BE`, the exact tag first. A request in `nl-NL` is boosted and
+sorted on the `nl` keys, and the GraphQL surface lists a document’s `nl` value
+first. A sibling region (`nl-BE` for `nl-NL`) matches only after the exact tag
+and the more general one. The exact match still wins where both are declared
+(`en` beside `en-GB`), and `und` matches nothing but untagged literals. Display fields are **omitted, never empty**, when a document
 lacks that language, and the per-locale search fields likewise, so declare them
 optional in the engine schema and sort with `missing_values: last`. A sort field
 is present whenever the field holds any value in a declared locale (the fallback

packages/search-api-graphql/src/language.ts

@@ -1,4 +1,5 @@
 import type { LocalizedValue } from '@lde/search';
+import { matchLanguage } from '@lde/search/adapter';
 
 /** One entry of the surface’s best-first `[LanguageString!]!`. `language` is null
  *  for untagged (`und`) values; `[0]` is the value to display and `[0].language`
@@ -15,12 +16,17 @@ export type LanguageOrder = (
 ) => readonly string[];
 
 /**
- * Default ordering: requested languages first (in request order), then the
- * remaining tagged languages, then untagged (`und`) last — so `[0]` is always the
- * best available value.
+ * Default ordering: requested languages first (in request order, each matched
+ * as a language range – `nl-NL` finds `nl` – most specific first), then the
+ * remaining tagged languages, then untagged (`und`) last – so `[0]` is always
+ * the best available value.
  */
 export const defaultLanguageOrder: LanguageOrder = (available, accept) => {
-  const requested = accept.filter((language) => available.includes(language));
+  const requested = [
+    ...new Set(
+      accept.flatMap((language) => matchLanguage(language, available)),
+    ),
+  ];
   const rest = available.filter(
     (language) => language !== 'und' && !requested.includes(language),
   );

packages/search-api-graphql/test/language.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+import { defaultLanguageOrder } from '../src/language.js';
+
+describe('defaultLanguageOrder', () => {
+  it('puts requested languages first, then the rest, then und', () => {
+    expect(defaultLanguageOrder(['und', 'nl', 'en', 'fr'], ['en'])).toEqual([
+      'en',
+      'nl',
+      'fr',
+      'und',
+    ]);
+  });
+
+  it('finds the primary language for a regional request', () => {
+    // A browser’s `nl-NL` must surface the Dutch value, not whatever language
+    // the document happens to list first.
+    expect(defaultLanguageOrder(['en', 'nl'], ['nl-NL'])).toEqual(['nl', 'en']);
+  });
+
+  it('finds regional values for a primary-language request', () => {
+    expect(defaultLanguageOrder(['en', 'nl-BE'], ['nl'])).toEqual([
+      'nl-BE',
+      'en',
+    ]);
+  });
+
+  it('ranks the exact tag before a general one, without duplicates', () => {
+    expect(
+      defaultLanguageOrder(['en', 'en-GB', 'nl'], ['en-GB', 'en', 'nl']),
+    ).toEqual(['en-GB', 'en', 'nl']);
+  });
+});

packages/search-typesense/src/query-compiler.ts

@@ -24,6 +24,7 @@ import {
   isWelded,
   joinGraph,
   localLookupTypeOf,
+  matchLanguage,
   pageForOffset,
   physicalFields,
   nestedReferenceType,
@@ -240,13 +241,14 @@ function collectSearchable(
     const baseWeight = field.searchable.weight;
     if (field.kind === 'text') {
       const locales = field.locales;
+      const activeLocale = matchLanguage(locale, locales)[0];
       search.forEach((name, index) => {
         names.push(qualify(prefix, name));
         // The active locale keeps full weight; `und` is language-neutral, so
         // it is never demoted (an untagged-only field would otherwise always
         // rank below its declared weight).
         weights.push(
-          locales[index] === locale || locales[index] === 'und'
+          locales[index] === activeLocale || locales[index] === 'und'
             ? baseWeight
             : Math.max(1, baseWeight - 1),
         );
@@ -645,9 +647,12 @@ function compileSort(
   }
   const field = fieldNamed(searchType, sort.field);
   if (field !== undefined && field.kind === 'text') {
-    // Sort on the active locale's key, falling back to the first declared
-    // locale (e.g. an und-only field sorted by a user in any language).
-    const index = field.locales.indexOf(locale);
+    // Sort on the key of the declared locale the active one best matches as a
+    // language range (`nl-NL` sorts on `nl`), falling back to the first
+    // declared locale (e.g. an und-only field sorted by a user in any language).
+    const activeLocale = matchLanguage(locale, field.locales)[0];
+    const index =
+      activeLocale === undefined ? -1 : field.locales.indexOf(activeLocale);
     const sortName = physicalFields(field).sort[index === -1 ? 0 : index];
     if (sortName !== undefined) {
       return `${sortName}:${sort.direction}`;

packages/search-typesense/test/query-compiler.test.ts

@@ -84,6 +84,28 @@ describe('buildSearchParams', () => {
     ).toBe('4,5,1');
   });
 
+  it('boosts the declared locale a regional tag falls under', () => {
+    expect(
+      buildSearchParams({ ...base, text: 'Kaart', locale: 'en-GB' }, schema)
+        .query_by_weights,
+    ).toBe('4,5,1');
+  });
+
+  it('sorts a regional tag on its declared locale’s key', () => {
+    // locales is ['nl', 'en']: without range matching `en-GB` would fall back
+    // to the first locale and sort English readers by the Dutch title.
+    expect(
+      buildSearchParams(
+        {
+          ...base,
+          locale: 'en-GB',
+          orderBy: [{ field: 'title', direction: 'asc' }],
+        },
+        schema,
+      ).sort_by,
+    ).toBe('title_sort_en:asc');
+  });
+
   it('maps offset/limit to numbered pages', () => {
     expect(
       buildSearchParams({ ...base, offset: 40, limit: 20 }, schema).page,

packages/search-typesense/vite.config.ts

@@ -28,7 +28,7 @@ export default mergeConfig(
           // projection naming what no lookup reaches are unreachable through
           // the port, since `assertValidQuery` rejects such a query first.
           // They hold for a direct caller, and are exercised as one.
-          branches: 95.89,
+          branches: 95.9,
           statements: 99.48,
         },
       },

packages/search/src/adapter.ts

@@ -47,6 +47,7 @@ export {
 } from './schema.js';
 export type { PhysicalFields } from './schema.js';
 export { physicalNameTokens } from './physical-name.js';
+export { matchLanguage } from './language.js';
 export { joinGraph, MAX_JOIN_DEPTH } from './join-graph.js';
 export type { JoinGraph } from './join-graph.js';
 export {

packages/search/src/language.ts

@@ -0,0 +1,67 @@
+/**
+ * The entries of `candidates` that language tag `tag` matches, best first, as
+ * the entries themselves (so a caller can index back into `candidates`).
+ *
+ * Tags are compared per BCP-47 subtag and case-insensitively, as language
+ * ranges (RFC 4647). A candidate matches when it shares the tag’s primary
+ * language, ranked:
+ *
+ * 1. the exact tag (`nl-NL` for `nl-NL`);
+ * 2. the tag’s truncations, most specific first – RFC 4647 lookup, so a
+ *    request for `nl-NL` finds a declared `nl`;
+ * 3. the tags it is a prefix of, in `candidates` order – RFC 4647 basic
+ *    filtering, so a range `nl` finds `nl-NL` and `nl-BE`;
+ * 4. the remaining tags of the same language, longest shared prefix first, so
+ *    a request for `nl-NL` still finds `nl-BE` rather than another language.
+ *
+ * The exact match therefore still wins where a general and a specific tag sit
+ * side by side (`en` and `en-GB`). `und`, the untagged bucket, matches nothing
+ * but `und`.
+ */
+export function matchLanguage(
+  tag: string,
+  candidates: readonly string[],
+): string[] {
+  const subtags = tag.toLowerCase().split('-');
+  const ranked: { candidate: string; kind: number; closeness: number }[] = [];
+  for (const candidate of candidates) {
+    const candidateSubtags = candidate.toLowerCase().split('-');
+    const shared = sharedPrefixLength(subtags, candidateSubtags);
+    const exact =
+      shared === subtags.length && shared === candidateSubtags.length;
+    if (shared === 0 || (subtags[0] === 'und' && !exact)) {
+      continue;
+    }
+    const kind = exact
+      ? 0
+      : shared === candidateSubtags.length
+        ? 1
+        : shared === subtags.length
+          ? 2
+          : 3;
+    // Extensions keep candidate order; truncations and siblings rank by how
+    // much of the tag they share.
+    ranked.push({ candidate, kind, closeness: kind === 2 ? 0 : shared });
+  }
+  return ranked
+    .sort(
+      (left, right) =>
+        left.kind - right.kind || right.closeness - left.closeness,
+    )
+    .map(({ candidate }) => candidate);
+}
+
+function sharedPrefixLength(
+  left: readonly string[],
+  right: readonly string[],
+): number {
+  let length = 0;
+  while (
+    length < left.length &&
+    length < right.length &&
+    left[length] === right[length]
+  ) {
+    length++;
+  }
+  return length;
+}

packages/search/src/project.ts

@@ -1,5 +1,6 @@
 import type { Quad } from '@rdfjs/types';
 import { fold } from '@lde/text-normalization';
+import { matchLanguage } from './language.js';
 import {
   buildSubjectIndex,
   frameSubjects,
@@ -562,11 +563,38 @@ function applyText(
   // here it simply indexes nothing.
   if (field.searchable !== undefined || field.sortable === true) {
     const names = physicalFields(field);
-    const valuesPerLocale = field.locales.map((locale) =>
-      values
-        .filter((value) => value.lang === locale)
-        .map((value) => value.value),
-    );
+    // A literal lands under the declared locale its tag best matches as a
+    // language range, so `nl-NL` titles are keyed under a declared `nl`; within
+    // a locale, the exact tag comes first.
+    const valuesPerLanguage = new Map<string, string[]>();
+    for (const { lang, value } of values) {
+      const languageValues = valuesPerLanguage.get(lang);
+      if (languageValues === undefined) {
+        valuesPerLanguage.set(lang, [value]);
+      } else {
+        languageValues.push(value);
+      }
+    }
+    const languagesPerLocale = new Map<string, [string, string[]][]>();
+    for (const entry of valuesPerLanguage) {
+      const locale = matchLanguage(entry[0], field.locales)[0];
+      if (locale !== undefined) {
+        languagesPerLocale.set(locale, [
+          ...(languagesPerLocale.get(locale) ?? []),
+          entry,
+        ]);
+      }
+    }
+    const valuesPerLocale = field.locales.map((locale) => {
+      const entries = languagesPerLocale.get(locale) ?? [];
+      const ranked = matchLanguage(
+        locale,
+        entries.map(([language]) => language),
+      );
+      return entries
+        .sort(([left], [right]) => ranked.indexOf(left) - ranked.indexOf(right))
+        .flatMap(([, languageValues]) => languageValues);
+    });
     // What every locale’s sort key falls back to: the document’s first value
     // in `locales` order, which is the declaration’s own statement of which
     // language stands in for the others.

packages/search/test/language.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest';
+import { matchLanguage } from '../src/language.js';
+
+describe('matchLanguage', () => {
+  it('matches an exact tag', () => {
+    expect(matchLanguage('nl', ['en', 'nl'])).toEqual(['nl']);
+  });
+
+  it('finds a declared primary subtag for a regional tag', () => {
+    expect(matchLanguage('nl-NL', ['en', 'nl'])).toEqual(['nl']);
+  });
+
+  it('finds regional tags for a primary subtag, in candidate order', () => {
+    expect(matchLanguage('nl', ['nl-NL', 'en', 'nl-BE'])).toEqual([
+      'nl-NL',
+      'nl-BE',
+    ]);
+  });
+
+  it('ranks the exact tag first, then truncations, then extensions', () => {
+    expect(
+      matchLanguage('zh-Hant', ['zh-Hant-TW', 'zh', 'zh-Hant', 'en']),
+    ).toEqual(['zh-Hant', 'zh', 'zh-Hant-TW']);
+    expect(matchLanguage('zh-Hant-TW', ['zh', 'zh-Hant'])).toEqual([
+      'zh-Hant',
+      'zh',
+    ]);
+  });
+
+  it('lets the exact tag win where general and specific sit side by side', () => {
+    expect(matchLanguage('en', ['en-GB', 'en'])).toEqual(['en', 'en-GB']);
+    expect(matchLanguage('en-GB', ['en', 'en-GB'])).toEqual(['en-GB', 'en']);
+  });
+
+  it('falls back to a sibling region after every closer match', () => {
+    expect(matchLanguage('nl-NL', ['nl-BE', 'en', 'nl'])).toEqual([
+      'nl',
+      'nl-BE',
+    ]);
+    expect(
+      matchLanguage('zh-Hant-TW', ['zh-Hans', 'zh-Hant-HK', 'zh-TW']),
+    ).toEqual(['zh-Hant-HK', 'zh-Hans', 'zh-TW']);
+  });
+
+  it('does not match another language', () => {
+    expect(matchLanguage('nl-NL', ['en', 'fr-BE'])).toEqual([]);
+  });
+
+  it('compares case-insensitively and returns the candidate as given', () => {
+    expect(matchLanguage('pt-br', ['PT', 'pt-BR'])).toEqual(['pt-BR', 'PT']);
+  });
+
+  it('compares whole subtags, not string prefixes', () => {
+    expect(matchLanguage('e', ['en'])).toEqual([]);
+  });
+
+  it('matches und only to und', () => {
+    expect(matchLanguage('und', ['nl', 'und'])).toEqual(['und']);
+    expect(matchLanguage('nl', ['und'])).toEqual([]);
+  });
+});

packages/search/test/project.test.ts

@@ -388,6 +388,37 @@ describe('projectDocument', () => {
     expect(document.title_sort_nl).toBeUndefined();
   });
 
+  it('indexes a regional tag under its declared primary language', () => {
+    const document = projectDocument(
+      {
+        '@id': 'https://ex/d/6a',
+        [dsKey('title')]: { '@language': 'nl-NL', '@value': 'Kaart' },
+      },
+      { name: 'Dataset', class: DATASET, fields },
+    );
+    // locales is ['nl', 'en']: `nl-NL` is Dutch, so it is searched and sorted
+    // under `nl` – and displayed under the tag the publisher gave it.
+    expect(document['title_nl-NL']).toBe('Kaart');
+    expect(document.title_search_nl).toBe('kaart');
+    expect(document.title_sort_nl).toBe('kaart');
+    expect(document.title_search_en).toBeUndefined();
+  });
+
+  it('puts the exact tag first when a regional one shares its locale', () => {
+    const document = projectDocument(
+      {
+        '@id': 'https://ex/d/6c',
+        [dsKey('title')]: [
+          { '@language': 'nl-BE', '@value': 'Plan' },
+          { '@language': 'nl', '@value': 'Kaart' },
+        ],
+      },
+      { name: 'Dataset', class: DATASET, fields },
+    );
+    expect(document.title_sort_nl).toBe('kaart');
+    expect(document.title_search_nl).toBe('kaart plan');
+  });
+
   it('normalises an underscore-style language tag to its BCP-47 shape', () => {
     const document = projectDocument(
       {

packages/search/vite.config.ts

@@ -12,7 +12,7 @@ export default mergeConfig(
         thresholds: {
           functions: 100,
           lines: 100,
-          branches: 99.72,
+          branches: 99.73,
           statements: 100,
         },
       },