diff --git a/CHANGELOG.md b/CHANGELOG.md index 2543889..11807ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Counts-only vocabulary dump** (`dumpVocabulary()`): inventories the vocabulary available in a pageset — labelled buttons, Grid 3 page WordLists, prediction dictionaries (`Prediction.PredictThis`) and smart-grammar word forms — returning counts per source with a part-of-speech breakdown. Emits counts only (never word lists), so it is safe for privacy-preserving reports. Exported from the `Analytics` and `Metrics` namespaces. +- `PagesetSummary.vocabulary`: optional `VocabularySummary` field so competence reports can carry the vocabulary inventory. +- CLI command `aac-processors vocabulary `: prints the vocabulary dump as JSON (`--out`, `--no-smart-grammar`, plus the usual filtering options). +- `AACPage.wordListItems` is now declared on the interface (it was only set at runtime by the Grid 3 processor). - **GoTalk NOW (`.gtbz`) support.** New `GotalkNowProcessor` reads Attainment Company's GoTalk NOW communication-board archives (ZIP of Apple plists + media). Implements `loadIntoTree`, `extractTexts`, `processTexts` diff --git a/src/cli/index.ts b/src/cli/index.ts index f8e8e2d..bdbc2dc 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -8,7 +8,8 @@ import { readGrid3History, readSnapUsage, } from '../utilities/analytics/history'; -import { ComparisonAnalyzer, MetricsCalculator } from '../utilities/analytics'; +import { ComparisonAnalyzer, MetricsCalculator, dumpVocabulary } from '../utilities/analytics'; +import type { VocabularyDump } from '../utilities/analytics'; import { CellScanningOrder, ScanningSelectionMethod } from '../types/aac'; import { defaultFileAdapter, extname } from '../utils/io'; import { readFileSync } from 'node:fs'; @@ -596,6 +597,87 @@ program } ); +program + .command('vocabulary ') + .description( + 'Counts-only vocabulary inventory: buttons, wordlists, prediction dictionaries and word forms (no word lists are emitted)' + ) + .option('--format ', 'Format type (auto-detected if not specified)') + .option('--out ', 'Write output to a file instead of stdout') + .option('--preserve-all-buttons', 'Preserve all buttons including navigation/system buttons') + .option('--no-exclude-navigation', "Don't exclude navigation buttons (Home, Back)") + .option('--no-exclude-system', "Don't exclude system buttons (Delete, Clear, etc.)") + .option('--exclude-buttons ', 'Comma-separated list of button labels/terms to exclude') + .option('--gridset-password ', 'Password for encrypted Grid3 archives (.gridsetx)') + .option('--no-smart-grammar', 'Skip smart-grammar word-form counts') + .action( + async ( + file: string, + options: { + format?: string; + out?: string; + preserveAllButtons?: boolean; + excludeNavigation?: boolean; + excludeSystem?: boolean; + excludeButtons?: string; + gridsetPassword?: string; + smartGrammar?: boolean; + } + ) => { + try { + const filteringOptions = parseFilteringOptions(options); + const format = options.format || (await detectFormat(file)); + const processor = getProcessor(format, filteringOptions); + + // The gridset loader logs debug chatter to stdout; silence it so the + // command's stdout is exactly one JSON document. + const silenced = Object.entries(console).reduce( + (acc, [k, fn]) => { + acc[k] = fn; + return acc; + }, + {} as Record + ); + for (const k of ['log', 'info', 'debug', 'warn']) { + (console as unknown as Record)[k] = () => {}; + } + let dump: VocabularyDump | undefined; + try { + const tree = await processor.loadIntoTree(file); + // analyze() first: it expands morphological predictions on the tree, + // which dumpVocabulary() needs for the word-form counts. + const metrics = new MetricsCalculator().analyze(tree, { + useSmartGrammar: options.smartGrammar, + }); + dump = dumpVocabulary(tree, { metrics }); + } finally { + for (const [k, fn] of Object.entries(silenced)) { + (console as unknown as Record)[k] = fn as (...a: unknown[]) => void; + } + } + + const result = { + format, + filtering: filteringOptions, + vocabulary: dump, + }; + + const output = JSON.stringify(result, null, 2); + if (options.out) { + await writeTextToPath(options.out, output); + } else { + console.log(output); + } + } catch (error) { + console.error( + 'Error dumping vocabulary:', + error instanceof Error ? error.message : String(error) + ); + process.exit(1); + } + } + ); + // Show help if no command provided if (process.argv.length <= 2) { program.help(); diff --git a/src/metrics.ts b/src/metrics.ts index 5ae2bc5..112e256 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -11,6 +11,13 @@ export * from './utilities/analytics/metrics/obl-types'; export { OblUtil, OblAnonymizer } from './utilities/analytics/metrics/obl'; export { MetricsCalculator } from './utilities/analytics/metrics/core'; export { VocabularyAnalyzer } from './utilities/analytics/metrics/vocabulary'; +export { dumpVocabulary } from './utilities/analytics/metrics/vocabularyDump'; +export type { + VocabularyDump, + VocabularyDumpOptions, + VocabularySourceCounts, + VocabularySummary, +} from './utilities/analytics/metrics/vocabularyDump'; export { SentenceAnalyzer } from './utilities/analytics/metrics/sentence'; export { ComparisonAnalyzer } from './utilities/analytics/metrics/comparison'; export { MorphologyEngine } from './utilities/analytics/morphology'; diff --git a/src/types/aac.ts b/src/types/aac.ts index 00c9b91..325b247 100644 --- a/src/types/aac.ts +++ b/src/types/aac.ts @@ -136,6 +136,9 @@ export interface AACPage { descriptionHtml?: string; images?: any[]; sounds?: any[]; + // Page-level WordList items (extracted from Grid3 ; surfaced + // regardless of whether AutoContent WordList cells exist in the grid). + wordListItems?: AACWordListItem[]; // Metrics support: Track semantic/clone IDs used on this page semantic_ids?: string[]; clone_ids?: string[]; diff --git a/src/utilities/analytics/competence.ts b/src/utilities/analytics/competence.ts index 2148545..8c82cdc 100644 --- a/src/utilities/analytics/competence.ts +++ b/src/utilities/analytics/competence.ts @@ -35,6 +35,8 @@ * reported only as a distribution. */ +import type { VocabularySummary } from './metrics/vocabularyDump'; + /** A single spoken utterance with a production timestamp (epoch ms). */ export interface CompetenceUtterance { text: string; @@ -443,6 +445,12 @@ export interface PagesetSummary { effort: DistributionStats; hasDynamicPrediction: boolean; spellingEffort: { base: number | null; perLetter: number | null }; + /** + * Counts-only vocabulary inventory (buttons, wordlists, prediction + * dictionaries, smart-grammar word forms) — see dumpVocabulary(). + * Null/undefined when the caller did not compute it. Contains counts only. + */ + vocabulary?: VocabularySummary | null; error?: string; } diff --git a/src/utilities/analytics/docs/VOCABULARY_ANALYSIS_GUIDE.md b/src/utilities/analytics/docs/VOCABULARY_ANALYSIS_GUIDE.md index 40cc71e..645c557 100644 --- a/src/utilities/analytics/docs/VOCABULARY_ANALYSIS_GUIDE.md +++ b/src/utilities/analytics/docs/VOCABULARY_ANALYSIS_GUIDE.md @@ -290,6 +290,81 @@ aac-processors coverage my-boardset.obf --core-lists default,unc --format markdo - `test-vocabulary-analysis.ts` - Vocabulary analysis demo - `test-comparison-analysis.ts` - Comparative analysis demo +## Vocabulary Dump (counts-only) + +For longitudinal metrics work (e.g. the Grid 3 competence reports), you also +want to know **how much vocabulary the user has available** — including the +wordlists and prediction dictionaries embedded in the grid, which button-label +counts alone miss. `dumpVocabulary()` inventories every vocabulary source and +returns counts only (never word lists), with a per-source part-of-speech +breakdown, so it is safe to embed in privacy-preserving reports. + +**Sources counted:** + +| Source | What it is | +| ----------------------- | -------------------------------------------------------------------------- | +| `buttons` | Labelled buttons (the static on-board vocabulary) | +| `wordLists` | Grid 3 page WordLists feeding dynamic AutoContent cells (``) | +| `predictionDictionaries`| Grid 3 prediction wordlists (`Prediction.PredictThis`) on prediction cells | +| `wordForms` | Smart-grammar inflections generated by `MetricsCalculator` (needs metrics) | + +**Usage:** + +```typescript +import { GridsetProcessor } from '@willwade/aac-processors/gridset'; +import { MetricsCalculator, dumpVocabulary } from '@willwade/aac-processors/metrics'; + +const processor = new GridsetProcessor(); +const tree = await processor.loadIntoTree('my.gridset'); +// analyze() BEFORE dumping: it expands morphological predictions on the tree. +const metrics = new MetricsCalculator().analyze(tree); +const dump = dumpVocabulary(tree, { metrics }); + +console.log(dump.summary.wordLists.lists, 'wordlists'); +console.log(dump.summary.wordLists.entries, 'wordlist words'); +console.log(dump.summary.predictionDictionaries.entries, 'dictionary words'); +console.log(dump.summary.wordForms?.entries ?? 0, 'smart-grammar inflections'); +console.log(dump.summary.combined.uniqueEntries, 'unique entries overall'); +``` + +**Output structure (counts only, no words):** + +```typescript +{ + schema: 'aac-vocabulary-dump/v1', + generatedAt: '2026-08-18T...', + source: { format: 'gridset', name: '...', locale: 'en-GB' }, + summary: { + totalBoards: 38, + totalButtons: 1147, + buttons: { entries, uniqueWords, uniquePhrases, byPartOfSpeech }, + wordLists: { entries, uniqueWords, uniquePhrases, byPartOfSpeech, lists: 19 }, + predictionDictionaries: { entries, ..., byPartOfSpeech, buttonsWithDictionaries }, + wordForms: { entries, ..., byPartOfSpeech, parentButtons } | null, + combined: { uniqueEntries, uniqueWords, uniquePhrases }, + }, +} +``` + +Notes: + +- Entries are normalised (lowercased, whitespace-collapsed); multi-word + entries ("thank you") are counted as phrases, not words. +- Word-form counts exclude the original dictionary words the inflections were + generated from (no double counting); `parameters.predictions` keeps the + originals even after `analyze()` rewrites `button.predictions`. +- Non-Grid 3 formats simply report zeros for the Grid 3-specific sources. + +**CLI:** + +```bash +# Counts-only vocabulary inventory as JSON +aac-processors vocabulary my.gridset --out vocabulary.json +``` + +The same summary is attached as `pageset.vocabulary` in the Grid 3 competence +report (`PagesetSummary.vocabulary`), and rendered in the exporter dashboard. + ## Morphological Vocabulary Coverage ### Problem diff --git a/src/utilities/analytics/index.ts b/src/utilities/analytics/index.ts index 3f9a935..ed7f999 100644 --- a/src/utilities/analytics/index.ts +++ b/src/utilities/analytics/index.ts @@ -29,6 +29,13 @@ export { MetricsCalculator } from './metrics/core'; // Export vocabulary and comparison analyzers export { VocabularyAnalyzer } from './metrics/vocabulary'; +export { dumpVocabulary } from './metrics/vocabularyDump'; +export type { + VocabularyDump, + VocabularyDumpOptions, + VocabularySourceCounts, + VocabularySummary, +} from './metrics/vocabularyDump'; export { SentenceAnalyzer } from './metrics/sentence'; export { ComparisonAnalyzer } from './metrics/comparison'; export { ReferenceLoader } from './reference'; diff --git a/src/utilities/analytics/metrics/vocabularyDump.ts b/src/utilities/analytics/metrics/vocabularyDump.ts new file mode 100644 index 0000000..4375484 --- /dev/null +++ b/src/utilities/analytics/metrics/vocabularyDump.ts @@ -0,0 +1,247 @@ +/** + * Vocabulary Dump (counts-only) + * + * Inventory of the vocabulary available in an AAC pageset, aggregated by + * source and part of speech. Only counts are emitted — never word lists — + * so the output is safe to embed in privacy-preserving reports. + * + * Sources counted: + * buttons — labelled buttons (the static on-board vocabulary) + * wordLists — page WordLists feeding dynamic AutoContent cells + * (Grid 3 ``; absent in other formats) + * predictionDictionaries — prediction wordlists attached to prediction cells + * (Grid 3 `Prediction.PredictThis` dictionaries) + * wordForms — smart-grammar inflections generated by + * MetricsCalculator (only when a MetricsResult is + * supplied) + * + * Non-Grid 3 formats simply report zeros for the Grid 3-specific sources. + */ + +import type { AACTree } from '../../../types/aac'; +import type { MetricsResult } from './types'; + +/** Normalise an entry: trim, lowercase, collapse internal whitespace. */ +function normalize(text: string): string { + return text.trim().toLowerCase().replace(/\s+/g, ' '); +} + +const UNTAGGED = 'Unknown'; + +/** Per-source tally accumulated while walking the tree. */ +class SourceTally { + entries = 0; + private words = new Set(); + private phrases = new Set(); + private byPos = new Map>(); + + add(text: string, pos?: string): void { + const norm = normalize(text); + if (!norm) return; + this.entries++; + if (/\s/.test(norm)) this.phrases.add(norm); + else this.words.add(norm); + const tag = pos && pos.trim() ? pos.trim() : UNTAGGED; + let set = this.byPos.get(tag); + if (!set) { + set = new Set(); + this.byPos.set(tag, set); + } + set.add(norm); + } + + counts(): VocabularySourceCounts { + const byPartOfSpeech: Record = {}; + for (const tag of Array.from(this.byPos.keys()).sort((a, b) => a.localeCompare(b))) { + byPartOfSpeech[tag] = this.byPos.get(tag)!.size; + } + return { + entries: this.entries, + uniqueWords: this.words.size, + uniquePhrases: this.phrases.size, + byPartOfSpeech, + }; + } + + get wordSet(): Set { + return this.words; + } + + get phraseSet(): Set { + return this.phrases; + } +} + +/** Counts for one vocabulary source. */ +export interface VocabularySourceCounts { + /** Total entries found in this source (before deduplication). */ + entries: number; + /** Unique single words after normalisation. */ + uniqueWords: number; + /** Unique multi-word phrases after normalisation (e.g. "thank you"). */ + uniquePhrases: number; + /** Unique entries per part-of-speech tag ('Unknown' when untagged). */ + byPartOfSpeech: Record; +} + +/** Aggregated, counts-only vocabulary inventory. */ +export interface VocabularySummary { + totalBoards: number; + totalButtons: number; + /** Vocabulary carried by labelled buttons. */ + buttons: VocabularySourceCounts; + /** Page WordLists (dynamic content cells; Grid 3). */ + wordLists: VocabularySourceCounts & { lists: number }; + /** Prediction dictionaries attached to prediction cells (Grid 3). */ + predictionDictionaries: VocabularySourceCounts & { buttonsWithDictionaries: number }; + /** Smart-grammar inflections; null when no MetricsResult was supplied. */ + wordForms: (VocabularySourceCounts & { parentButtons: number }) | null; + /** Unique entries across all sources combined. */ + combined: { + uniqueEntries: number; + uniqueWords: number; + uniquePhrases: number; + }; +} + +/** Full vocabulary dump document (schema-versioned). */ +export interface VocabularyDump { + schema: 'aac-vocabulary-dump/v1'; + generatedAt: string; + source: { format?: string; name?: string; locale?: string }; + summary: VocabularySummary; +} + +export interface VocabularyDumpOptions { + /** + * Precomputed metrics (MetricsCalculator.analyze). Enables the + * smart-grammar word-form counts. Run analyze() BEFORE dumpVocabulary(): + * analyze() expands morphological predictions on the tree. + */ + metrics?: MetricsResult; +} + +/** + * Count the vocabulary available in a pageset, by source and part of speech. + * Emits counts only — no word lists. + * + * @example + * const processor = new GridsetProcessor(); + * const tree = await processor.loadIntoTree('my.gridset'); + * const metrics = new MetricsCalculator().analyze(tree); + * const dump = dumpVocabulary(tree, { metrics }); + * console.log(dump.summary.wordLists.lists, 'wordlists'); + */ +export function dumpVocabulary(tree: AACTree, options?: VocabularyDumpOptions): VocabularyDump { + const pages = Object.values(tree.pages); + + const buttonTally = new SourceTally(); + const wordListTally = new SourceTally(); + const predictionTally = new SourceTally(); + let wordListCount = 0; + let buttonsWithDictionaries = 0; + let totalButtons = 0; + + for (const page of pages) { + totalButtons += page.buttons.length; + + if (page.wordListItems && page.wordListItems.length > 0) { + wordListCount++; + for (const item of page.wordListItems) { + wordListTally.add(item.text, item.partOfSpeech); + } + } + + for (const btn of page.buttons) { + const label = btn.label?.trim(); + if (label) { + buttonTally.add(label, (btn as { pos?: string }).pos); + } + + // Prediction dictionaries: parameters.predictions keeps the original + // Prediction.PredictThis words even after analyze() expands + // btn.predictions with morphological forms. + const dict = (btn.parameters as { predictions?: unknown } | undefined)?.predictions; + if (Array.isArray(dict) && dict.length > 0) { + buttonsWithDictionaries++; + for (const w of dict) { + if (typeof w === 'string') { + predictionTally.add(w, (btn as { pos?: string }).pos); + } + } + } + } + } + + // Smart-grammar word forms: inflected forms generated by MetricsCalculator + // (is_word_form buttons), excluding the original dictionary words they + // were generated from. + let wordForms: VocabularySummary['wordForms'] = null; + let formTally: SourceTally | null = null; + if (options?.metrics) { + const originals = new Set(); + for (const page of pages) { + for (const btn of page.buttons) { + const dict = (btn.parameters as { predictions?: unknown } | undefined)?.predictions; + if (Array.isArray(dict)) { + for (const w of dict) { + if (typeof w === 'string') { + const norm = normalize(w); + if (norm) originals.add(norm); + } + } + } + } + } + formTally = new SourceTally(); + const parents = new Set(); + const seen = new Set(); + for (const b of options.metrics.buttons) { + if (!b.is_word_form) continue; + const norm = normalize(b.label); + if (!norm || seen.has(norm) || originals.has(norm)) continue; + seen.add(norm); + formTally.add(b.label, b.pos); + if (b.parent_button_id) parents.add(b.parent_button_id); + } + wordForms = { ...formTally.counts(), parentButtons: parents.size }; + } + + const combinedWords = new Set([ + ...buttonTally.wordSet, + ...wordListTally.wordSet, + ...predictionTally.wordSet, + ...(formTally ? formTally.wordSet : []), + ]); + const combinedPhrases = new Set([ + ...buttonTally.phraseSet, + ...wordListTally.phraseSet, + ...predictionTally.phraseSet, + ]); + + return { + schema: 'aac-vocabulary-dump/v1', + generatedAt: new Date().toISOString(), + source: { + format: tree.metadata?.format, + name: tree.metadata?.name, + locale: tree.metadata?.locale, + }, + summary: { + totalBoards: pages.length, + totalButtons, + buttons: buttonTally.counts(), + wordLists: { ...wordListTally.counts(), lists: wordListCount }, + predictionDictionaries: { + ...predictionTally.counts(), + buttonsWithDictionaries, + }, + wordForms, + combined: { + uniqueEntries: combinedWords.size + combinedPhrases.size, + uniqueWords: combinedWords.size, + uniquePhrases: combinedPhrases.size, + }, + }, + }; +} diff --git a/test/cli.comprehensive.test.ts b/test/cli.comprehensive.test.ts index 7560574..a0f9b2f 100644 --- a/test/cli.comprehensive.test.ts +++ b/test/cli.comprehensive.test.ts @@ -351,6 +351,29 @@ describe('CLI Comprehensive Tests', () => { expect(result2).toContain('Home'); expect(result2).toContain('Food'); }); + it('should dump vocabulary counts for a gridset (counts only, no words)', async () => { + const exampleGridset = path.join(__dirname, '../test/assets/gridset/example.gridset'); + + if (fs.existsSync(exampleGridset)) { + const result = execSync(`node ${cliPath} vocabulary ${exampleGridset}`, { + encoding: 'utf8', + cwd: tempDir, + }); + + const parsed = JSON.parse(result); + expect(parsed.format).toBe('gridset'); + expect(parsed.vocabulary.schema).toBe('aac-vocabulary-dump/v1'); + const summary = parsed.vocabulary.summary; + expect(summary.totalBoards).toBeGreaterThan(10); + expect(summary.wordLists.lists).toBeGreaterThanOrEqual(15); + expect(summary.combined.uniqueEntries).toBeGreaterThan(15); + expect(summary.wordForms).not.toBeNull(); + // Counts only: no word arrays anywhere in the output. + expect(result).not.toContain('"missing_words"'); + } else { + console.log('Skipping test - example.gridset not found'); + } + }); }); describe('Error Handling Tests', () => { diff --git a/test/vocabularyDump.test.ts b/test/vocabularyDump.test.ts new file mode 100644 index 0000000..0d9023c --- /dev/null +++ b/test/vocabularyDump.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it } from '@jest/globals'; +import path from 'path'; +import { + dumpVocabulary, + type VocabularyDump, +} from '../src/utilities/analytics/metrics/vocabularyDump'; +import type { AACTree, AACButton } from '../src/types/aac'; +import type { MetricsResult } from '../src/utilities/analytics/metrics/types'; + +/* ------------------------------------------------------------------ * + * Helpers + * ------------------------------------------------------------------ */ + +function makeButton(id: string, label: string, extra: Partial = {}): AACButton { + return { id, label, message: label, ...extra }; +} + +function makeTree( + pages: Array<{ + id: string; + name: string; + buttons: AACButton[]; + wordListItems?: Array<{ text: string; image?: string; partOfSpeech?: string }>; + }> +): AACTree { + const pageMap: AACTree['pages'] = {}; + for (const p of pages) { + pageMap[p.id] = { + id: p.id, + name: p.name, + grid: [], + buttons: p.buttons, + parentId: null, + ...(p.wordListItems ? { wordListItems: p.wordListItems } : {}), + } as AACTree['pages'][string]; + } + return { + pages: pageMap, + metadata: { format: 'gridset', name: 'test set', locale: 'en-GB' }, + rootId: pages[0]?.id ?? null, + toolbarId: null, + getPage: (id: string) => pageMap[id], + addPage: () => {}, + } as unknown as AACTree; +} + +function makeMetrics(buttons: Partial[]): MetricsResult { + return { + analysis_version: 'test', + locale: 'en-GB', + total_boards: 1, + total_buttons: buttons.length, + total_words: buttons.length, + reference_counts: {}, + grid: { rows: 3, columns: 3 }, + buttons: buttons as MetricsResult['buttons'], + levels: {}, + }; +} + +/* ------------------------------------------------------------------ * + * Synthetic tree + * ------------------------------------------------------------------ */ + +const TREE = makeTree([ + { + id: 'home', + name: 'Home', + buttons: [ + makeButton('b1', 'hello', { pos: 'Verb' } as Partial & { pos?: string }), + makeButton('b2', 'thank you'), + makeButton('b3', 'want', { + parameters: { predictions: ['want', 'go'] }, + }), + ], + wordListItems: [ + { text: 'dog', partOfSpeech: 'Noun' }, + { text: 'cat' }, // untagged -> Unknown + { text: 'happy dog', partOfSpeech: 'Noun' }, // multi-word phrase + ], + }, +]); + +describe('dumpVocabulary', () => { + it('counts button vocabulary with POS breakdown', () => { + const dump = dumpVocabulary(TREE); + const b = dump.summary.buttons; + expect(b.entries).toBe(3); // hello, thank you, want + expect(b.uniqueWords).toBe(2); // hello, want + expect(b.uniquePhrases).toBe(1); // thank you + expect(b.byPartOfSpeech['Verb']).toBe(1); + expect(b.byPartOfSpeech['Unknown']).toBe(2); // thank you, want + expect(dump.summary.totalButtons).toBe(3); + expect(dump.summary.totalBoards).toBe(1); + }); + + it('counts page wordlists with per-list detail', () => { + const dump = dumpVocabulary(TREE); + const wl = dump.summary.wordLists; + expect(wl.lists).toBe(1); + expect(wl.entries).toBe(3); + expect(wl.uniqueWords).toBe(2); // dog, cat + expect(wl.uniquePhrases).toBe(1); // happy dog + expect(wl.byPartOfSpeech['Noun']).toBe(2); // dog, happy dog + expect(wl.byPartOfSpeech['Unknown']).toBe(1); // cat + }); + + it('counts prediction dictionaries from parameters.predictions', () => { + const dump = dumpVocabulary(TREE); + const pd = dump.summary.predictionDictionaries; + expect(pd.buttonsWithDictionaries).toBe(1); + expect(pd.entries).toBe(2); // want, go + expect(pd.uniqueWords).toBe(2); + }); + + it('returns null wordForms when no metrics supplied', () => { + const dump = dumpVocabulary(TREE); + expect(dump.summary.wordForms).toBeNull(); + }); + + it('counts smart-grammar word forms, excluding original dictionary words', () => { + const metrics = makeMetrics([ + { label: 'going', is_word_form: true, pos: 'Verb', parent_button_id: 'b1' }, + { label: 'want', is_word_form: true, pos: 'Verb', parent_button_id: 'b3' }, // original -> excluded + { label: 'hello', is_word_form: false }, + ]); + const dump = dumpVocabulary(TREE, { metrics }); + const wf = dump.summary.wordForms!; + expect(wf).not.toBeNull(); + expect(wf.entries).toBe(1); // going only + expect(wf.byPartOfSpeech['Verb']).toBe(1); + expect(wf.parentButtons).toBe(1); + }); + + it('combines unique entries across all sources', () => { + const metrics = makeMetrics([ + { label: 'going', is_word_form: true, pos: 'Verb', parent_button_id: 'b1' }, + ]); + const dump = dumpVocabulary(TREE, { metrics }); + // words: hello, want, dog, cat, go, going = 6; phrases: thank you, happy dog = 2 + expect(dump.summary.combined.uniqueWords).toBe(6); + expect(dump.summary.combined.uniquePhrases).toBe(2); + expect(dump.summary.combined.uniqueEntries).toBe(8); + }); + + it('is case-insensitive and whitespace-normalising', () => { + const tree = makeTree([ + { + id: 'p', + name: 'P', + buttons: [makeButton('a', ' Hello '), makeButton('b', 'HELLO')], + wordListItems: [{ text: 'thank you' }], + }, + ]); + const dump = dumpVocabulary(tree); + expect(dump.summary.buttons.entries).toBe(2); + expect(dump.summary.buttons.uniqueWords).toBe(1); + expect(dump.summary.wordLists.uniquePhrases).toBe(1); + }); + + it('emits schema/source metadata and no word lists', () => { + const dump: VocabularyDump = dumpVocabulary(TREE); + expect(dump.schema).toBe('aac-vocabulary-dump/v1'); + expect(dump.generatedAt).toBeTruthy(); + expect(dump.source.format).toBe('gridset'); + expect(dump.source.locale).toBe('en-GB'); + // Counts only: no word arrays anywhere in the output. + const json = JSON.stringify(dump); + expect(json).not.toContain('"dog"'); + expect(json).not.toContain('["hello"'); + }); +}); + +/* ------------------------------------------------------------------ * + * Real gridset asset + * ------------------------------------------------------------------ */ + +describe('dumpVocabulary on example.gridset', () => { + it('surfaces the embedded wordlists and full-pipeline word forms', async () => { + const { GridsetProcessor } = await import('../src/processors/gridsetProcessor'); + const { MetricsCalculator } = await import('../src/utilities/analytics/metrics/core'); + const assetPath = path.join(__dirname, 'assets/gridset/example.gridset'); + + const processor = new GridsetProcessor(); + const tree = await processor.loadIntoTree(assetPath); + const metrics = new MetricsCalculator().analyze(tree); + + const dump = dumpVocabulary(tree, { metrics }); + const s = dump.summary; + + expect(s.totalBoards).toBeGreaterThan(10); + // This asset embeds 15 non-empty page wordlists (38 more are empty placeholders). + expect(s.wordLists.lists).toBeGreaterThanOrEqual(15); + expect(s.wordLists.entries).toBeGreaterThan(15); + expect(s.combined.uniqueEntries).toBeGreaterThan(100); + // Smart grammar should generate at least some inflections (POS-tagged set). + expect(s.wordForms).not.toBeNull(); + }, 30000); +});