From cd43aefd1ac98fcf19983eae65210da00ee8eee3 Mon Sep 17 00:00:00 2001 From: jstet Date: Thu, 24 Sep 2026 14:34:06 +0200 Subject: [PATCH] fix(from_file): convert registered vocabularies in the browser; keep quoted labels select_*_from_file only converted to LimeSurvey TSV when a Node caller read registry/vocab/.csv from disk and passed it in, so the browser entry threw "Unimplemented XLSForm type" for a type the registry lists as supported. - codegen emits src/generated/VocabularyOptions.ts: every registered vocabulary's options, parsed with Python's csv module. - src/vocab.ts (browser-safe): registeredFileChoices() and parseVocabCsv(), now on the RFC 4180 tokenizer from responseFile.ts. parseVocabCsv is exported from the package root. - XLSFormToTSVConverter.convert() inlines registered vocabularies by default; an explicit fileChoices entry adds an unregistered file or overrides a registered one. XLSFormParser.convertXLS{File,Data}ToTSV take fileChoices. - fileChoices.ts (node-only) keeps only the from-disk lookup the CLI uses for CSVs beside a form. The old parseVocabCsv split on every comma, so 14 ISO 3166-1 labels such as "Korea, Republic of" were emitted as '"Korea'. The two long_list TSV snapshots are re-blessed; that is their only change. The bless script and the snapshot contract test no longer pass file choices, so they exercise the built-in path. The generated skill told authors to put from_file options on the choices sheet; it now points them to the registered-vocabulary table instead. Closes #24 Co-Authored-By: Claude Opus 5.5 (1M context) --- ARCHITECTURE.md | 5 +- codegen/cli.py | 4 + codegen/emit_skill.py | 9 +- codegen/emit_ts.py | 41 +++ codegen/loader.py | 5 +- .../select_multiple_long_list/tsv.tsv | 28 +- .../entities/select_one_long_list/tsv.tsv | 28 +- scripts/bless-tsv-snapshots.mjs | 5 - .../references/question-types.md | 4 +- src/fileChoices.ts | 52 +--- src/generated/VocabularyOptions.ts | 260 ++++++++++++++++++ src/index.ts | 1 + src/pipelines/xlsform2lstsv/constants.ts | 4 +- src/pipelines/xlsform2lstsv/index.ts | 12 +- src/responseFile.ts | 2 +- src/vocab.ts | 73 +++++ src/xlsform/parser.ts | 21 +- tests/ts/contract/tsvSnapshots.test.ts | 21 +- tests/ts/integration/browserBundle.test.ts | 40 ++- tests/ts/unit/vocab.test.ts | 98 +++++++ 20 files changed, 599 insertions(+), 114 deletions(-) create mode 100644 src/generated/VocabularyOptions.ts create mode 100644 src/vocab.ts create mode 100644 tests/ts/unit/vocab.test.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index a373982..9aae3a0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -58,7 +58,10 @@ Kobo or LimeSurvey is the platform's own export step (see Node-only, and never imported by `src/index.ts`: `src/cli.ts` (the `formtransform` binary), `src/cliShared.ts` (its argument parsing and file I/O), `src/fileChoices.ts` (loads `select_*_from_file` CSVs from disk) and -`src/generateFixtures.ts`. +`src/generateFixtures.ts`. The registered vocabularies don't need +`fileChoices.ts`: codegen emits their options to +`src/generated/VocabularyOptions.ts`, and the converter inlines them in the +browser too. ### Code Generation (`codegen/`) diff --git a/codegen/cli.py b/codegen/cli.py index 3f954a4..5b4f916 100644 --- a/codegen/cli.py +++ b/codegen/cli.py @@ -12,6 +12,7 @@ generate_question_types, generate_typescript, generate_typescript_ddi, + generate_vocabulary_options, ) from .examples import build_example_artifacts from .loader import load_registry @@ -66,6 +67,9 @@ def main(argv: list[str] | None = None) -> None: generate_question_types(registry, ts_out / "QuestionTypes.ts") print(" ✅ src/generated/QuestionTypes.ts (labelled catalogue for consumers)") + generate_vocabulary_options(registry, base_dir / "registry" / "vocab", ts_out / "VocabularyOptions.ts") + print(" ✅ src/generated/VocabularyOptions.ts (registered vocabularies for select_*_from_file)") + generate_schematron(registry, base_dir / "ddi-validation" / "schematron" / "ddi_custom_rules.sch") print(" ✅ ddi-validation/schematron/ddi_custom_rules.sch (DDI validation rules)") diff --git a/codegen/emit_skill.py b/codegen/emit_skill.py index f739c11..9cb9a3a 100644 --- a/codegen/emit_skill.py +++ b/codegen/emit_skill.py @@ -174,7 +174,14 @@ def _type_section(d: dict[str, Any], variants: list[dict], base_dir: Path) -> li aliases = d["xlsform"].get("aliases") if aliases: out.append(f"**Accepted aliases:** {', '.join(f'`{a}`' for a in aliases)}\n") - if d["xlsform"].get("requiresListName"): + if d.get("ddi", {}).get("externalCodeList"): + out.append( + "**Requires a registered vocabulary:** write the type cell as " + f"`{ts} .csv` with a file from the registered-vocabularies table in " + "[xlsform-syntax.md](xlsform-syntax.md). The options come from that " + "vocabulary; add no rows to the `choices` sheet.\n" + ) + elif d["xlsform"].get("requiresListName"): out.append( "**Requires a choice list:** write the type cell as " f"`{ts} ` and define the options on the `choices` sheet under " diff --git a/codegen/emit_ts.py b/codegen/emit_ts.py index bc9b3aa..d644aa2 100644 --- a/codegen/emit_ts.py +++ b/codegen/emit_ts.py @@ -1,5 +1,6 @@ """Generators that emit TypeScript / JSON artifacts consumed by the in-repo transformer.""" +import csv import json from collections import OrderedDict from pathlib import Path @@ -353,6 +354,46 @@ def generate_conventions_ts(registry: dict[str, Any], output: Path): ) +def generate_vocabulary_options(registry: dict[str, Any], vocab_dir: Path, output: Path): + """Emit every registered vocabulary's options as a TypeScript module. + + `select_*_from_file` inlines the vocabulary's options into LimeSurvey TSV. + The browser has no filesystem to read `registry/vocab/.csv` from, so + the registered vocabularies ship as data (formtransform#24). Parsed with the + `csv` module: labels such as "Korea, Republic of" are quoted. + """ + lines = [ + "/**", + " * GENERATED CODE - DO NOT EDIT", + " * Source: registry/root.jsonld + registry/vocab/*.csv", + " * Generated by: codegen", + " */", + "", + "/** Registered vocabulary options as `[code, label]`, keyed by xlsform filename. */", + "export const VOCABULARY_OPTIONS: Record = {", + ] + vocabs = sorted( + ( + d["xlsformFilename"] + for d in registry.values() + if d.get("@type") == "Vocabulary" and d.get("xlsformFilename") + ), + ) + for filename in vocabs: + with open(vocab_dir / filename, encoding="utf-8-sig", newline="") as f: + rows = list(csv.DictReader(f)) + lines.append(f" {_ts_key(filename)}: [") + for row in rows: + code = (row.get("code") or "").strip() + if code: + label = (row.get("label") or "").strip() + lines.append(f" [{json.dumps(code, ensure_ascii=False)}, {json.dumps(label, ensure_ascii=False)}],") + lines.append(" ],") + lines.append("};") + lines.append("") + output.write_text("\n".join(lines)) + + def _entry_lines( key: str, *, diff --git a/codegen/loader.py b/codegen/loader.py index 8e95b69..cbffe72 100644 --- a/codegen/loader.py +++ b/codegen/loader.py @@ -56,9 +56,8 @@ def _validate_vocab_files(registry: dict[str, Any], vocab_dir: Path) -> None: Vocabularies are open-ended (see convention:externalCodeList's ``vocabularyDeclaration``): declaring one is the only step, so this is the check that keeps a declaration honest. Without it a typo'd or missing CSV - surfaces much later — the forward converter silently falls back to erroring - on an "unimplemented" from_file type when `resolveFileChoices` finds no - options to inline. + surfaces much later, as a missing file when `VocabularyOptions.ts` is + generated. """ errors: list[str] = [] for at_id, data in registry.items(): diff --git a/registry/entities/select_multiple_long_list/tsv.tsv b/registry/entities/select_multiple_long_list/tsv.tsv index 90d4422..a387e1f 100644 --- a/registry/entities/select_multiple_long_list/tsv.tsv +++ b/registry/entities/select_multiple_long_list/tsv.tsv @@ -31,8 +31,8 @@ SQ BZ Belize en SQ BJ Benin en SQ BM Bermuda en SQ BT Bhutan en -SQ BO "Bolivia en -SQ BQ "Bonaire en +SQ BO Bolivia, Plurinational State of en +SQ BQ Bonaire, Sint Eustatius and Saba en SQ BA Bosnia and Herzegovina en SQ BW Botswana en SQ BV Bouvet Island en @@ -56,7 +56,7 @@ SQ CC Cocos (Keeling) Islands en SQ CO Colombia en SQ KM Comoros en SQ CG Congo en -SQ CD "Congo en +SQ CD Congo, Democratic Republic of the en SQ CK Cook Islands en SQ CR Costa Rica en SQ CI Côte d'Ivoire en @@ -110,7 +110,7 @@ SQ HU Hungary en SQ IS Iceland en SQ IN India en SQ ID Indonesia en -SQ IR "Iran en +SQ IR Iran, Islamic Republic of en SQ IQ Iraq en SQ IE Ireland en SQ IM Isle of Man en @@ -123,8 +123,8 @@ SQ JO Jordan en SQ KZ Kazakhstan en SQ KE Kenya en SQ KI Kiribati en -SQ KP "Korea en -SQ KR "Korea en +SQ KP Korea, Democratic People's Republic of en +SQ KR Korea, Republic of en SQ KW Kuwait en SQ KG Kyrgyzstan en SQ LA Lao People's Democratic Republic en @@ -149,8 +149,8 @@ SQ MR Mauritania en SQ MU Mauritius en SQ YT Mayotte en SQ MX Mexico en -SQ FM "Micronesia en -SQ MD "Moldova en +SQ FM Micronesia, Federated States of en +SQ MD Moldova, Republic of en SQ MC Monaco en SQ MN Mongolia en SQ ME Montenegro en @@ -161,7 +161,7 @@ SQ MM Myanmar en SQ NA Namibia en SQ NR Nauru en SQ NP Nepal en -SQ NL "Netherlands en +SQ NL Netherlands, Kingdom of the en SQ NC New Caledonia en SQ NZ New Zealand en SQ NI Nicaragua en @@ -175,7 +175,7 @@ SQ NO Norway en SQ OM Oman en SQ PK Pakistan en SQ PW Palau en -SQ PS "Palestine en +SQ PS Palestine, State of en SQ PA Panama en SQ PG Papua New Guinea en SQ PY Paraguay en @@ -191,7 +191,7 @@ SQ RO Romania en SQ RU Russian Federation en SQ RW Rwanda en SQ BL Saint Barthélemy en -SQ SH "Saint Helena en +SQ SH Saint Helena, Ascension and Tristan da Cunha en SQ KN Saint Kitts and Nevis en SQ LC Saint Lucia en SQ MF Saint Martin (French part) en @@ -222,9 +222,9 @@ SQ SJ Svalbard and Jan Mayen en SQ SE Sweden en SQ CH Switzerland en SQ SY Syrian Arab Republic en -SQ TW "Taiwan en +SQ TW Taiwan, Province of China en SQ TJ Tajikistan en -SQ TZ "Tanzania en +SQ TZ Tanzania, United Republic of en SQ TH Thailand en SQ TL Timor-Leste en SQ TG Togo en @@ -245,7 +245,7 @@ SQ UM United States Minor Outlying Islands en SQ UY Uruguay en SQ UZ Uzbekistan en SQ VU Vanuatu en -SQ VE "Venezuela en +SQ VE Venezuela, Bolivarian Republic of en SQ VN Viet Nam en SQ VG Virgin Islands (British) en SQ VI Virgin Islands (U.S.) en diff --git a/registry/entities/select_one_long_list/tsv.tsv b/registry/entities/select_one_long_list/tsv.tsv index 2c1a89f..cc9d31a 100644 --- a/registry/entities/select_one_long_list/tsv.tsv +++ b/registry/entities/select_one_long_list/tsv.tsv @@ -31,8 +31,8 @@ A BZ Belize en A BJ Benin en A BM Bermuda en A BT Bhutan en -A BO "Bolivia en -A BQ "Bonaire en +A BO Bolivia, Plurinational State of en +A BQ Bonaire, Sint Eustatius and Saba en A BA Bosnia and Herzegovina en A BW Botswana en A BV Bouvet Island en @@ -56,7 +56,7 @@ A CC Cocos (Keeling) Islands en A CO Colombia en A KM Comoros en A CG Congo en -A CD "Congo en +A CD Congo, Democratic Republic of the en A CK Cook Islands en A CR Costa Rica en A CI Côte d'Ivoire en @@ -110,7 +110,7 @@ A HU Hungary en A IS Iceland en A IN India en A ID Indonesia en -A IR "Iran en +A IR Iran, Islamic Republic of en A IQ Iraq en A IE Ireland en A IM Isle of Man en @@ -123,8 +123,8 @@ A JO Jordan en A KZ Kazakhstan en A KE Kenya en A KI Kiribati en -A KP "Korea en -A KR "Korea en +A KP Korea, Democratic People's Republic of en +A KR Korea, Republic of en A KW Kuwait en A KG Kyrgyzstan en A LA Lao People's Democratic Republic en @@ -149,8 +149,8 @@ A MR Mauritania en A MU Mauritius en A YT Mayotte en A MX Mexico en -A FM "Micronesia en -A MD "Moldova en +A FM Micronesia, Federated States of en +A MD Moldova, Republic of en A MC Monaco en A MN Mongolia en A ME Montenegro en @@ -161,7 +161,7 @@ A MM Myanmar en A NA Namibia en A NR Nauru en A NP Nepal en -A NL "Netherlands en +A NL Netherlands, Kingdom of the en A NC New Caledonia en A NZ New Zealand en A NI Nicaragua en @@ -175,7 +175,7 @@ A NO Norway en A OM Oman en A PK Pakistan en A PW Palau en -A PS "Palestine en +A PS Palestine, State of en A PA Panama en A PG Papua New Guinea en A PY Paraguay en @@ -191,7 +191,7 @@ A RO Romania en A RU Russian Federation en A RW Rwanda en A BL Saint Barthélemy en -A SH "Saint Helena en +A SH Saint Helena, Ascension and Tristan da Cunha en A KN Saint Kitts and Nevis en A LC Saint Lucia en A MF Saint Martin (French part) en @@ -222,9 +222,9 @@ A SJ Svalbard and Jan Mayen en A SE Sweden en A CH Switzerland en A SY Syrian Arab Republic en -A TW "Taiwan en +A TW Taiwan, Province of China en A TJ Tajikistan en -A TZ "Tanzania en +A TZ Tanzania, United Republic of en A TH Thailand en A TL Timor-Leste en A TG Togo en @@ -245,7 +245,7 @@ A UM United States Minor Outlying Islands en A UY Uruguay en A UZ Uzbekistan en A VU Vanuatu en -A VE "Venezuela en +A VE Venezuela, Bolivarian Republic of en A VN Viet Nam en A VG Virgin Islands (British) en A VI Virgin Islands (U.S.) en diff --git a/scripts/bless-tsv-snapshots.mjs b/scripts/bless-tsv-snapshots.mjs index 60ce32a..d3c069d 100644 --- a/scripts/bless-tsv-snapshots.mjs +++ b/scripts/bless-tsv-snapshots.mjs @@ -22,10 +22,6 @@ if (!fs.existsSync(entry)) { process.exit(1); } const { XLSFormToTSVConverter } = await import(pathToFileURL(entry).href); -const { resolveFileChoices } = await import( - pathToFileURL(path.join(REPO_ROOT, 'dist', 'fileChoices.js')).href -); -const VOCAB_DIR = path.join(REPO_ROOT, 'registry', 'vocab'); // Same registry walk as codegen.load_registry: root graph + every split source // (schema.jsonld + conventions/*.jsonld + registry//definition.jsonld). @@ -68,7 +64,6 @@ for (const variant of variants) { survey, xlsform.choices ?? [], xlsform.settings ?? [], - resolveFileChoices(survey, VOCAB_DIR), ); fs.writeFileSync(path.join(exDir, 'tsv.tsv'), tsv); console.log( diff --git a/skills/cdl-survey-types/references/question-types.md b/skills/cdl-survey-types/references/question-types.md index 81a9826..985fc39 100644 --- a/skills/cdl-survey-types/references/question-types.md +++ b/skills/cdl-survey-types/references/question-types.md @@ -49,7 +49,7 @@ **Use when:** When selecting from a long non-exclusive list of options maintained as a controlled vocabulary, allowing multiple selections. -**Requires a choice list:** write the type cell as `select_multiple_from_file ` and define the options on the `choices` sheet under that `list_name`. +**Requires a registered vocabulary:** write the type cell as `select_multiple_from_file .csv` with a file from the registered-vocabularies table in [xlsform-syntax.md](xlsform-syntax.md). The options come from that vocabulary; add no rows to the `choices` sheet. **Constraints:** variable `name` ≤ 20 chars, `^[a-zA-Z0-9]+$` @@ -72,7 +72,7 @@ **Use when:** When selecting from a long closed list of exhaustive, mutually exclusive options maintained as a controlled vocabulary (e.g. country, occupation), presented via dropdown or autocomplete. -**Requires a choice list:** write the type cell as `select_one_from_file ` and define the options on the `choices` sheet under that `list_name`. +**Requires a registered vocabulary:** write the type cell as `select_one_from_file .csv` with a file from the registered-vocabularies table in [xlsform-syntax.md](xlsform-syntax.md). The options come from that vocabulary; add no rows to the `choices` sheet. **Constraints:** variable `name` ≤ 20 chars, `^[a-zA-Z0-9]+$` diff --git a/src/fileChoices.ts b/src/fileChoices.ts index f378ce2..77c8d82 100644 --- a/src/fileChoices.ts +++ b/src/fileChoices.ts @@ -1,62 +1,30 @@ /** * Filesystem resolver for `select_*_from_file .csv` choice lists. * - * The converter core is pure (no fs); this node-only helper loads the CSV files - * a survey references and returns them keyed by filename, ready to pass as + * The converter core is pure (no fs) and already knows the registered + * vocabularies (see `vocab.ts`). This node-only helper loads CSV files that sit + * beside a form on disk, e.g. an unregistered vocabulary, and returns them + * keyed by filename, ready to pass as * `convert(survey, choices, settings, fileChoices)`. Kept OUT of the browser - * entry (src/index) — import it directly (CLI / bless scripts / tests). - * - * Each referenced CSV must have a `code,label` header (e.g. the registered - * controlled vocabularies under registry/vocab/). Rows become ChoiceRow - * `{ list_name: , name: , label: