diff --git a/CHANGELOG.md b/CHANGELOG.md index c350a79..4b46ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Lexical richness indices** (`lexicalRichness()`): Brunet's W (`N·V^-0.165`, lower = richer) and Honoré's R (`100·ln N/(1−V₁/V)`, higher = richer) plus hapax-legomena count — total-sample type-token measures that complement MATTR without window-size sensitivity (used in DEPAC's lexical-complexity feature set). Language-agnostic, no resources needed; attached to each `MonthBin.lexicalRichness` in competence reports. - **Counts-only vocabulary dump** (`dumpVocabulary()`): inventories the vocabulary available in a pageset — labelled buttons, Grid 3 page WordLists, prediction dictionaries (`Prediction.PredictThis`) and smart-grammar word forms — returning counts per source with a part-of-speech breakdown. Emits counts only (never word lists), so it is safe for privacy-preserving reports. Exported from the `Analytics` and `Metrics` namespaces. - `PagesetSummary.vocabulary`: optional `VocabularySummary` field so competence reports can carry the vocabulary inventory. - CLI command `aac-processors vocabulary `: prints the vocabulary dump as JSON (`--out`, `--no-smart-grammar`, plus the usual filtering options). diff --git a/src/utilities/analytics/competence.ts b/src/utilities/analytics/competence.ts index 8c82cdc..aef48db 100644 --- a/src/utilities/analytics/competence.ts +++ b/src/utilities/analytics/competence.ts @@ -341,6 +341,44 @@ export function spellingValidity(words: WordStream, dictionary?: Set): n return checked === 0 ? null : correct / checked; } +/* ------------------------------------------------------------------ * + * Lexical richness indices (Brunet's W, Honoré's R) + * + * Total-sample type-token measures that complement MATTR: they do not depend + * on a moving window, so they behave differently on small/variable AAC + * samples. Both are pure functions of the token-frequency distribution — + * language-agnostic, no word lists needed. Used in DEPAC's lexical-complexity + * feature set (Tasnim et al., 2022); lower W / higher R = richer vocabulary. + * ------------------------------------------------------------------ */ + +export interface LexicalRichness { + /** Brunet's index W = N · V^(-0.165). Range ~10–30; LOWER = richer. */ + brunetsW: number | null; + /** Honoré's statistic R = 100·ln N / (1 − V1/V). HIGHER = richer. */ + honoresR: number | null; + /** Hapax legomena (words used exactly once) — count only. */ + hapax: number; + /** Number of distinct words (V). */ + types: number; + /** Number of tokens (N). */ + tokens: number; +} + +export function lexicalRichness(words: WordStream): LexicalRichness { + const n = words.length; + const freq = new Map(); + for (const w of words) freq.set(w, (freq.get(w) ?? 0) + 1); + const v = freq.size; + let v1 = 0; + for (const c of freq.values()) if (c === 1) v1++; + + const brunetsW = n > 0 && v > 0 ? n * Math.pow(v, -0.165) : null; + // Honoré's R is undefined when every word is a hapax (V1 = V) or N <= 1. + const honoresR = n > 1 && v > 0 && v1 < v ? (100 * Math.log(n)) / (1 - v1 / v) : null; + + return { brunetsW, honoresR, hapax: v1, types: v, tokens: n }; +} + /* ------------------------------------------------------------------ * * Activity / engagement statistics * ------------------------------------------------------------------ */ @@ -413,6 +451,8 @@ export interface MonthBin { morphologicalDiversity: DiversityResult; /** Phonological — only when a dictionary is supplied. */ spellingValidity: number | null; + /** Lexical richness indices (Brunet's W, Honoré's R) — always available. */ + lexicalRichness: LexicalRichness; /** True when the month has too little data to trust the diversity figures. */ suppressed: boolean; suppressReason: string | null; @@ -635,6 +675,9 @@ export function analyzeTimeline( classifyInflection: resources.classifyInflection, }); const spell = suppressed ? null : spellingValidity(stream, dictionary); + const rich: LexicalRichness = suppressed + ? { brunetsW: null, honoresR: null, hapax: 0, types: 0, tokens: 0 } + : lexicalRichness(stream); timeline.push({ month: key, @@ -647,6 +690,7 @@ export function analyzeTimeline( syntacticDiversity: syn, morphologicalDiversity: mor, spellingValidity: spell, + lexicalRichness: rich, suppressed, suppressReason, }); diff --git a/test/competence.test.ts b/test/competence.test.ts index a053f27..8c0e180 100644 --- a/test/competence.test.ts +++ b/test/competence.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from '@jest/globals'; import { analyzeTimeline, lexicalDiversity, + lexicalRichness, morphologicalDiversity, movingAverageTTR, spellingValidity, @@ -197,6 +198,38 @@ describe('competence / summarizeActivity', () => { }); }); +describe('competence / lexicalRichness (Brunet W, Honoré R)', () => { + it('computes W, R and hapax counts for a mixed stream', () => { + // N=6, V=5 ("the" twice + cat/dog/bird/fish once each) -> V1=4 + const r = lexicalRichness(['the', 'cat', 'the', 'dog', 'bird', 'fish']); + expect(r.tokens).toBe(6); + expect(r.types).toBe(5); + expect(r.hapax).toBe(4); + expect(r.brunetsW).toBeCloseTo(6 * Math.pow(5, -0.165), 10); + expect(r.honoresR).toBeCloseTo((100 * Math.log(6)) / (1 - 4 / 5), 10); + }); + + it('returns the closed-form Honoré R for a standard example', () => { + // N=5, V=4, V1=3: "a a b c d" + const r = lexicalRichness(['a', 'a', 'b', 'c', 'd']); + expect(r.hapax).toBe(3); + expect(r.honoresR).toBeCloseTo((100 * Math.log(5)) / (1 - 3 / 4), 10); + }); + + it('R is null when every type is a hapax or N<=1', () => { + expect(lexicalRichness(['a', 'b', 'c']).honoresR).toBeNull(); + expect(lexicalRichness(['a']).honoresR).toBeNull(); + expect(lexicalRichness([]).brunetsW).toBeNull(); + expect(lexicalRichness([]).honoresR).toBeNull(); + }); + + it('W decreases (richer) as vocabulary diversifies at fixed N', () => { + const narrow = lexicalRichness(['go', 'go', 'go', 'go', 'go', 'go']); + const wide = lexicalRichness(['go', 'went', 'going', 'goes', 'gone', 'go']); + expect(wide.brunetsW!).toBeLessThan(narrow.brunetsW!); + }); +}); + describe('competence / analyzeTimeline', () => { const DAY = 86_400_000; @@ -229,6 +262,10 @@ describe('competence / analyzeTimeline', () => { for (const bin of report.timeline) { expect(bin.suppressed).toBe(false); expect(bin.lexicalDiversity.median).not.toBeNull(); + // Lexical richness is resource-free — always present on unsuppressed bins. + expect(bin.lexicalRichness.brunetsW).not.toBeNull(); + expect(bin.lexicalRichness.tokens).toBe(bin.words); + expect(bin.lexicalRichness.types).toBe(bin.uniqueWords); } });