Linguistic competence metrics + counts-only vocabulary dump

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- **Counts-only vocabulary dump** (`dumpVocabulary()`): inventories the vocabulary available in a pageset — labelled buttons, Grid 3 page WordLists, prediction dictionaries (`Prediction.PredictThis`) and smart-grammar word forms — returning counts per source with a part-of-speech breakdown. Emits counts only (never word lists), so it is safe for privacy-preserving reports. Exported from the `Analytics` and `Metrics` namespaces.
+- `PagesetSummary.vocabulary`: optional `VocabularySummary` field so competence reports can carry the vocabulary inventory.
+- CLI command `aac-processors vocabulary <file>`: prints the vocabulary dump as JSON (`--out`, `--no-smart-grammar`, plus the usual filtering options).
+- `AACPage.wordListItems` is now declared on the interface (it was only set at runtime by the Grid 3 processor).
 - **GoTalk NOW (`.gtbz`) support.** New `GotalkNowProcessor` reads Attainment
   Company's GoTalk NOW communication-board archives (ZIP of Apple plists +
   media). Implements `loadIntoTree`, `extractTexts`, `processTexts`

src/cli/index.ts

@@ -8,7 +8,8 @@ import {
   readGrid3History,
   readSnapUsage,
 } from '../utilities/analytics/history';
-import { ComparisonAnalyzer, MetricsCalculator } from '../utilities/analytics';
+import { ComparisonAnalyzer, MetricsCalculator, dumpVocabulary } from '../utilities/analytics';
+import type { VocabularyDump } from '../utilities/analytics';
 import { CellScanningOrder, ScanningSelectionMethod } from '../types/aac';
 import { defaultFileAdapter, extname } from '../utils/io';
 import { readFileSync } from 'node:fs';
@@ -596,6 +597,87 @@ program
     }
   );
 
+program
+  .command('vocabulary <file>')
+  .description(
+    'Counts-only vocabulary inventory: buttons, wordlists, prediction dictionaries and word forms (no word lists are emitted)'
+  )
+  .option('--format <format>', 'Format type (auto-detected if not specified)')
+  .option('--out <path>', 'Write output to a file instead of stdout')
+  .option('--preserve-all-buttons', 'Preserve all buttons including navigation/system buttons')
+  .option('--no-exclude-navigation', "Don't exclude navigation buttons (Home, Back)")
+  .option('--no-exclude-system', "Don't exclude system buttons (Delete, Clear, etc.)")
+  .option('--exclude-buttons <list>', 'Comma-separated list of button labels/terms to exclude')
+  .option('--gridset-password <password>', 'Password for encrypted Grid3 archives (.gridsetx)')
+  .option('--no-smart-grammar', 'Skip smart-grammar word-form counts')
+  .action(
+    async (
+      file: string,
+      options: {
+        format?: string;
+        out?: string;
+        preserveAllButtons?: boolean;
+        excludeNavigation?: boolean;
+        excludeSystem?: boolean;
+        excludeButtons?: string;
+        gridsetPassword?: string;
+        smartGrammar?: boolean;
+      }
+    ) => {
+      try {
+        const filteringOptions = parseFilteringOptions(options);
+        const format = options.format || (await detectFormat(file));
+        const processor = getProcessor(format, filteringOptions);
+
+        // The gridset loader logs debug chatter to stdout; silence it so the
+        // command's stdout is exactly one JSON document.
+        const silenced = Object.entries(console).reduce(
+          (acc, [k, fn]) => {
+            acc[k] = fn;
+            return acc;
+          },
+          {} as Record<string, unknown>
+        );
+        for (const k of ['log', 'info', 'debug', 'warn']) {
+          (console as unknown as Record<string, unknown>)[k] = () => {};
+        }
+        let dump: VocabularyDump | undefined;
+        try {
+          const tree = await processor.loadIntoTree(file);
+          // analyze() first: it expands morphological predictions on the tree,
+          // which dumpVocabulary() needs for the word-form counts.
+          const metrics = new MetricsCalculator().analyze(tree, {
+            useSmartGrammar: options.smartGrammar,
+          });
+          dump = dumpVocabulary(tree, { metrics });
+        } finally {
+          for (const [k, fn] of Object.entries(silenced)) {
+            (console as unknown as Record<string, unknown>)[k] = fn as (...a: unknown[]) => void;
+          }
+        }
+
+        const result = {
+          format,
+          filtering: filteringOptions,
+          vocabulary: dump,
+        };
+
+        const output = JSON.stringify(result, null, 2);
+        if (options.out) {
+          await writeTextToPath(options.out, output);
+        } else {
+          console.log(output);
+        }
+      } catch (error) {
+        console.error(
+          'Error dumping vocabulary:',
+          error instanceof Error ? error.message : String(error)
+        );
+        process.exit(1);
+      }
+    }
+  );
+
 // Show help if no command provided
 if (process.argv.length <= 2) {
   program.help();

src/metrics.ts

@@ -11,6 +11,13 @@ export * from './utilities/analytics/metrics/obl-types';
 export { OblUtil, OblAnonymizer } from './utilities/analytics/metrics/obl';
 export { MetricsCalculator } from './utilities/analytics/metrics/core';
 export { VocabularyAnalyzer } from './utilities/analytics/metrics/vocabulary';
+export { dumpVocabulary } from './utilities/analytics/metrics/vocabularyDump';
+export type {
+  VocabularyDump,
+  VocabularyDumpOptions,
+  VocabularySourceCounts,
+  VocabularySummary,
+} from './utilities/analytics/metrics/vocabularyDump';
 export { SentenceAnalyzer } from './utilities/analytics/metrics/sentence';
 export { ComparisonAnalyzer } from './utilities/analytics/metrics/comparison';
 export { MorphologyEngine } from './utilities/analytics/morphology';

src/types/aac.ts

@@ -136,6 +136,9 @@ export interface AACPage {
   descriptionHtml?: string;
   images?: any[];
   sounds?: any[];
+  // Page-level WordList items (extracted from Grid3 <WordList>; surfaced
+  // regardless of whether AutoContent WordList cells exist in the grid).
+  wordListItems?: AACWordListItem[];
   // Metrics support: Track semantic/clone IDs used on this page
   semantic_ids?: string[];
   clone_ids?: string[];

src/utilities/analytics/competence.ts

@@ -35,6 +35,8 @@
  * reported only as a distribution.
  */
 
+import type { VocabularySummary } from './metrics/vocabularyDump';
+
 /** A single spoken utterance with a production timestamp (epoch ms). */
 export interface CompetenceUtterance {
   text: string;
@@ -443,6 +445,12 @@ export interface PagesetSummary {
   effort: DistributionStats;
   hasDynamicPrediction: boolean;
   spellingEffort: { base: number | null; perLetter: number | null };
+  /**
+   * Counts-only vocabulary inventory (buttons, wordlists, prediction
+   * dictionaries, smart-grammar word forms) — see dumpVocabulary().
+   * Null/undefined when the caller did not compute it. Contains counts only.
+   */
+  vocabulary?: VocabularySummary | null;
   error?: string;
 }
 

src/utilities/analytics/docs/VOCABULARY_ANALYSIS_GUIDE.md

@@ -290,6 +290,81 @@ aac-processors coverage my-boardset.obf --core-lists default,unc --format markdo
 - `test-vocabulary-analysis.ts` - Vocabulary analysis demo
 - `test-comparison-analysis.ts` - Comparative analysis demo
 
+## Vocabulary Dump (counts-only)
+
+For longitudinal metrics work (e.g. the Grid 3 competence reports), you also
+want to know **how much vocabulary the user has available** — including the
+wordlists and prediction dictionaries embedded in the grid, which button-label
+counts alone miss. `dumpVocabulary()` inventories every vocabulary source and
+returns counts only (never word lists), with a per-source part-of-speech
+breakdown, so it is safe to embed in privacy-preserving reports.
+
+**Sources counted:**
+
+| Source                  | What it is                                                                 |
+| ----------------------- | -------------------------------------------------------------------------- |
+| `buttons`               | Labelled buttons (the static on-board vocabulary)                           |
+| `wordLists`             | Grid 3 page WordLists feeding dynamic AutoContent cells (`<WordList>`)      |
+| `predictionDictionaries`| Grid 3 prediction wordlists (`Prediction.PredictThis`) on prediction cells  |
+| `wordForms`             | Smart-grammar inflections generated by `MetricsCalculator` (needs metrics)  |
+
+**Usage:**
+
+```typescript
+import { GridsetProcessor } from '@willwade/aac-processors/gridset';
+import { MetricsCalculator, dumpVocabulary } from '@willwade/aac-processors/metrics';
+
+const processor = new GridsetProcessor();
+const tree = await processor.loadIntoTree('my.gridset');
+// analyze() BEFORE dumping: it expands morphological predictions on the tree.
+const metrics = new MetricsCalculator().analyze(tree);
+const dump = dumpVocabulary(tree, { metrics });
+
+console.log(dump.summary.wordLists.lists, 'wordlists');
+console.log(dump.summary.wordLists.entries, 'wordlist words');
+console.log(dump.summary.predictionDictionaries.entries, 'dictionary words');
+console.log(dump.summary.wordForms?.entries ?? 0, 'smart-grammar inflections');
+console.log(dump.summary.combined.uniqueEntries, 'unique entries overall');
+```
+
+**Output structure (counts only, no words):**
+
+```typescript
+{
+  schema: 'aac-vocabulary-dump/v1',
+  generatedAt: '2026-08-18T...',
+  source: { format: 'gridset', name: '...', locale: 'en-GB' },
+  summary: {
+    totalBoards: 38,
+    totalButtons: 1147,
+    buttons:              { entries, uniqueWords, uniquePhrases, byPartOfSpeech },
+    wordLists:            { entries, uniqueWords, uniquePhrases, byPartOfSpeech, lists: 19 },
+    predictionDictionaries: { entries, ..., byPartOfSpeech, buttonsWithDictionaries },
+    wordForms:            { entries, ..., byPartOfSpeech, parentButtons } | null,
+    combined: { uniqueEntries, uniqueWords, uniquePhrases },
+  },
+}
+```
+
+Notes:
+
+- Entries are normalised (lowercased, whitespace-collapsed); multi-word
+  entries ("thank you") are counted as phrases, not words.
+- Word-form counts exclude the original dictionary words the inflections were
+  generated from (no double counting); `parameters.predictions` keeps the
+  originals even after `analyze()` rewrites `button.predictions`.
+- Non-Grid 3 formats simply report zeros for the Grid 3-specific sources.
+
+**CLI:**
+
+```bash
+# Counts-only vocabulary inventory as JSON
+aac-processors vocabulary my.gridset --out vocabulary.json
+```
+
+The same summary is attached as `pageset.vocabulary` in the Grid 3 competence
+report (`PagesetSummary.vocabulary`), and rendered in the exporter dashboard.
+
 ## Morphological Vocabulary Coverage
 
 ### Problem

src/utilities/analytics/index.ts

@@ -29,6 +29,13 @@ export { MetricsCalculator } from './metrics/core';
 
 // Export vocabulary and comparison analyzers
 export { VocabularyAnalyzer } from './metrics/vocabulary';
+export { dumpVocabulary } from './metrics/vocabularyDump';
+export type {
+  VocabularyDump,
+  VocabularyDumpOptions,
+  VocabularySourceCounts,
+  VocabularySummary,
+} from './metrics/vocabularyDump';
 export { SentenceAnalyzer } from './metrics/sentence';
 export { ComparisonAnalyzer } from './metrics/comparison';
 export { ReferenceLoader } from './reference';