Add lexical richness indices (Brunet's W, Honoré's R) to competence timeline

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Lexical richness indices** (`lexicalRichness()`): Brunet's W (`N·V^-0.165`, lower = richer) and Honoré's R (`100·ln N/(1−V₁/V)`, higher = richer) plus hapax-legomena count — total-sample type-token measures that complement MATTR without window-size sensitivity (used in DEPAC's lexical-complexity feature set). Language-agnostic, no resources needed; attached to each `MonthBin.lexicalRichness` in competence reports.
 - **Counts-only vocabulary dump** (`dumpVocabulary()`): inventories the vocabulary available in a pageset — labelled buttons, Grid 3 page WordLists, prediction dictionaries (`Prediction.PredictThis`) and smart-grammar word forms — returning counts per source with a part-of-speech breakdown. Emits counts only (never word lists), so it is safe for privacy-preserving reports. Exported from the `Analytics` and `Metrics` namespaces.
 - `PagesetSummary.vocabulary`: optional `VocabularySummary` field so competence reports can carry the vocabulary inventory.
 - CLI command `aac-processors vocabulary <file>`: prints the vocabulary dump as JSON (`--out`, `--no-smart-grammar`, plus the usual filtering options).

src/utilities/analytics/competence.ts

@@ -341,6 +341,44 @@ export function spellingValidity(words: WordStream, dictionary?: Set<string>): n
   return checked === 0 ? null : correct / checked;
 }
 
+/* ------------------------------------------------------------------ *
+ * Lexical richness indices (Brunet's W, Honoré's R)
+ *
+ * Total-sample type-token measures that complement MATTR: they do not depend
+ * on a moving window, so they behave differently on small/variable AAC
+ * samples. Both are pure functions of the token-frequency distribution —
+ * language-agnostic, no word lists needed. Used in DEPAC's lexical-complexity
+ * feature set (Tasnim et al., 2022); lower W / higher R = richer vocabulary.
+ * ------------------------------------------------------------------ */
+
+export interface LexicalRichness {
+  /** Brunet's index W = N · V^(-0.165). Range ~10–30; LOWER = richer. */
+  brunetsW: number | null;
+  /** Honoré's statistic R = 100·ln N / (1 − V1/V). HIGHER = richer. */
+  honoresR: number | null;
+  /** Hapax legomena (words used exactly once) — count only. */
+  hapax: number;
+  /** Number of distinct words (V). */
+  types: number;
+  /** Number of tokens (N). */
+  tokens: number;
+}
+
+export function lexicalRichness(words: WordStream): LexicalRichness {
+  const n = words.length;
+  const freq = new Map<string, number>();
+  for (const w of words) freq.set(w, (freq.get(w) ?? 0) + 1);
+  const v = freq.size;
+  let v1 = 0;
+  for (const c of freq.values()) if (c === 1) v1++;
+
+  const brunetsW = n > 0 && v > 0 ? n * Math.pow(v, -0.165) : null;
+  // Honoré's R is undefined when every word is a hapax (V1 = V) or N <= 1.
+  const honoresR = n > 1 && v > 0 && v1 < v ? (100 * Math.log(n)) / (1 - v1 / v) : null;
+
+  return { brunetsW, honoresR, hapax: v1, types: v, tokens: n };
+}
+
 /* ------------------------------------------------------------------ *
  * Activity / engagement statistics
  * ------------------------------------------------------------------ */
@@ -413,6 +451,8 @@ export interface MonthBin {
   morphologicalDiversity: DiversityResult;
   /** Phonological — only when a dictionary is supplied. */
   spellingValidity: number | null;
+  /** Lexical richness indices (Brunet's W, Honoré's R) — always available. */
+  lexicalRichness: LexicalRichness;
   /** True when the month has too little data to trust the diversity figures. */
   suppressed: boolean;
   suppressReason: string | null;
@@ -635,6 +675,9 @@ export function analyzeTimeline(
           classifyInflection: resources.classifyInflection,
         });
     const spell = suppressed ? null : spellingValidity(stream, dictionary);
+    const rich: LexicalRichness = suppressed
+      ? { brunetsW: null, honoresR: null, hapax: 0, types: 0, tokens: 0 }
+      : lexicalRichness(stream);
 
     timeline.push({
       month: key,
@@ -647,6 +690,7 @@ export function analyzeTimeline(
       syntacticDiversity: syn,
       morphologicalDiversity: mor,
       spellingValidity: spell,
+      lexicalRichness: rich,
       suppressed,
       suppressReason,
     });

test/competence.test.ts

@@ -2,6 +2,7 @@
 import {
   analyzeTimeline,
   lexicalDiversity,
+  lexicalRichness,
   morphologicalDiversity,
   movingAverageTTR,
   spellingValidity,
@@ -197,6 +198,38 @@
   });
 });
 
+describe('competence / lexicalRichness (Brunet W, Honoré R)', () => {
+  it('computes W, R and hapax counts for a mixed stream', () => {
+    // N=6, V=5 ("the" twice + cat/dog/bird/fish once each) -> V1=4
+    const r = lexicalRichness(['the', 'cat', 'the', 'dog', 'bird', 'fish']);
+    expect(r.tokens).toBe(6);
+    expect(r.types).toBe(5);
+    expect(r.hapax).toBe(4);
+    expect(r.brunetsW).toBeCloseTo(6 * Math.pow(5, -0.165), 10);
+    expect(r.honoresR).toBeCloseTo((100 * Math.log(6)) / (1 - 4 / 5), 10);
+  });
+
+  it('returns the closed-form Honoré R for a standard example', () => {
+    // N=5, V=4, V1=3: "a a b c d"
+    const r = lexicalRichness(['a', 'a', 'b', 'c', 'd']);
+    expect(r.hapax).toBe(3);
+    expect(r.honoresR).toBeCloseTo((100 * Math.log(5)) / (1 - 3 / 4), 10);
+  });
+
+  it('R is null when every type is a hapax or N<=1', () => {
+    expect(lexicalRichness(['a', 'b', 'c']).honoresR).toBeNull();
+    expect(lexicalRichness(['a']).honoresR).toBeNull();
+    expect(lexicalRichness([]).brunetsW).toBeNull();
+    expect(lexicalRichness([]).honoresR).toBeNull();
+  });
+
+  it('W decreases (richer) as vocabulary diversifies at fixed N', () => {
+    const narrow = lexicalRichness(['go', 'go', 'go', 'go', 'go', 'go']);
+    const wide = lexicalRichness(['go', 'went', 'going', 'goes', 'gone', 'go']);
+    expect(wide.brunetsW!).toBeLessThan(narrow.brunetsW!);
+  });
+});
+
 describe('competence / analyzeTimeline', () => {
   const DAY = 86_400_000;
 
@@ -229,6 +262,10 @@
     for (const bin of report.timeline) {
       expect(bin.suppressed).toBe(false);
       expect(bin.lexicalDiversity.median).not.toBeNull();
+      // Lexical richness is resource-free — always present on unsuppressed bins.
+      expect(bin.lexicalRichness.brunetsW).not.toBeNull();
+      expect(bin.lexicalRichness.tokens).toBe(bin.words);
+      expect(bin.lexicalRichness.types).toBe(bin.uniqueWords);
     }
   });