From 0609b26ec69b5800846207f57004cc165b6f83cd Mon Sep 17 00:00:00 2001 From: jstet Date: Thu, 24 Sep 2026 14:43:54 +0200 Subject: [PATCH] test(live): port survey2ddi's qwacback DDI equivalence test tests/live/qwacback/ sends the same 15 XLSForm cases through buildDdiXml and qwacback's POST /api/convert/xlsform-to-ddi and compares the / shapes, with the helpers and cases carried over from survey2ddi's tests/integration/test_conversion_equivalence.py. It stays opt-in: it lives under tests/live/ and is auto-marked `docker`. - Own compose file with qwacback only; conversion doesn't need the worker. QWACBACK_IMAGE overrides the image (ghcr.io/correlaid/qwacback is private), QWACBACK_URL reuses a running instance, and the tests skip with the reason when the image can't be pulled. - build_ddi.mjs calls buildDdiXml from dist/ over stdin/stdout. - Backs off on 429 (qwacback allows guests 10 conversions a minute). Against qwacback main (c99de96): 13 pass, 2 strict xfails. - range: unregistered here, so emitted as text; qwacback emits numeric (#33). - note: by design formtransform emits no for a note. tests/README.md said qwacback already consumed this library's DDI emitter; it still runs its own Go converter. HANDOVER_QWAC.md records the two differences qwacback's converter swap will cause. Closes #14 Co-Authored-By: Claude Opus 5.5 (1M context) --- HANDOVER_QWAC.md | 14 +- tests/README.md | 27 +- tests/live/qwacback/build_ddi.mjs | 8 + tests/live/qwacback/conftest.py | 120 +++++++ tests/live/qwacback/docker-compose.yml | 23 ++ .../qwacback/test_qwacback_equivalence.py | 296 ++++++++++++++++++ 6 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 tests/live/qwacback/build_ddi.mjs create mode 100644 tests/live/qwacback/conftest.py create mode 100644 tests/live/qwacback/docker-compose.yml create mode 100644 tests/live/qwacback/test_qwacback_equivalence.py diff --git a/HANDOVER_QWAC.md b/HANDOVER_QWAC.md index 6b9b6f7..3b209a2 100644 --- a/HANDOVER_QWAC.md +++ b/HANDOVER_QWAC.md @@ -128,9 +128,17 @@ pinned to one version. The plan holds, but: `registry/entities//fixtures/xlsform.json`, not `registry/types//examples//xlsform.json`. And `lstsv2ddi` / `lstsv2xlsform` now exist, which the brief predates. -- **Coverage handoff:** survey2ddi's qwacback equivalence test is being ported - into formtransform (formtransform#14). qwacback's own converter tests go - with the Go converter. +- **Coverage handoff:** survey2ddi's qwacback equivalence test now lives in + formtransform (`tests/live/qwacback/`, formtransform#14). Against qwacback + `main` (`c99de96`), 13 of 15 types match. The two that differ change + qwacback's DDI when it swaps converters: + - `range`: qwacback emits numeric/`contin`, formtransform text (unregistered + type, formtransform#33). + - `note`: qwacback emits a ``, formtransform none. This is intended: a + note stores no response, so formtransform folds it into + ``/``. + + qwacback's own converter tests go with the Go converter. ## Hard rules for whoever does the work diff --git a/tests/README.md b/tests/README.md index 745c257..981e5a7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -25,10 +25,12 @@ everything that needs a JVM, an external oracle, or a running engine. | XLSForm fixture inputs are valid XLSForm | external oracle ([pyxform](https://github.com/XLSForm/pyxform)) | pytest — `tests/validation/test_xlsform_pyxform.py` | | LimeSurvey accepts the blessed TSV snapshots | live LimeSurvey (docker) | pytest — `tests/live/limesurvey/test_registry_entities.py` | | What LimeSurvey *stores* when a respondent answers | live LimeSurvey (docker) + Playwright | pytest — `tests/live/limesurvey/test_response_roundtrip.py` | +| qwacback's Go converter emits the same DDI shape as `buildDdiXml` | live qwacback (docker) | pytest — `tests/live/qwacback/test_qwacback_equivalence.py` | -Downstream `qwacback` (Go) is no longer conformance-tested here: it consumes -this library's DDI emitter through `@correlaid/formtransform`, installed from -GitHub, rather than maintaining its own converter. +qwacback still runs its own Go XLSForm → DDI converter. It plans to replace it +with this library ([HANDOVER_QWAC.md](../HANDOVER_QWAC.md)), and the +equivalence test is the parity check for that swap. After the swap it compares +the library with itself and can go. ## Layout @@ -62,6 +64,10 @@ tests/ answers/.json # what the respondent enters expected/.json # blessed exported response output/ # generated TSVs (gitignored) + qwacback/ + docker-compose.yml # qwacback alone (QWACBACK_IMAGE; the ghcr image is private) + test_qwacback_equivalence.py # same XLSForm → buildDdiXml vs. qwacback, DDI shape compared + build_ddi.mjs # buildDdiXml from dist/ over stdin/stdout fixtures/surveys// # one folder per whole survey, like a registry entity xlsform.json | xlsform.xlsx # authored source tsv.tsv ddi.xml # blessed forward snapshots @@ -175,6 +181,21 @@ one — a note stores nothing, and the blessed snapshot records that). Needs node + Playwright/Chromium on top of the docker stack; skips if Playwright is not resolvable (locally or globally). +### qwacback equivalence (`tests/live/qwacback/test_qwacback_equivalence.py`) + +Ported from survey2ddi (formtransform#14). For every answer type qwacback +supports, the same XLSForm goes through `buildDdiXml` and qwacback's +`POST /api/convert/xlsform-to-ddi`, and the ``/`` shapes are +compared. qwacback returns a bare `` or `` when there's only one, +so the test wraps it in a ``. Two cases are strict xfails: `range` +(#33) and `note` (by design: formtransform emits no `` for a note). + +The fixture starts qwacback from its own compose file, or uses `QWACBACK_URL`. +`ghcr.io/correlaid/qwacback` is private: log in to ghcr.io, or build it +(`docker build -t qwacback:local ../qwacback`) and set +`QWACBACK_IMAGE=qwacback:local`. If the image can't be pulled, the tests skip +with that reason. + ## Running ```bash diff --git a/tests/live/qwacback/build_ddi.mjs b/tests/live/qwacback/build_ddi.mjs new file mode 100644 index 0000000..a1359bf --- /dev/null +++ b/tests/live/qwacback/build_ddi.mjs @@ -0,0 +1,8 @@ +// Reads {"survey": [...], "choices": {...}} on stdin and prints the DDI XML +// that formtransform's buildDdiXml produces, using the built dist/. +import { buildDdiXml } from '../../../dist/index.js'; + +let input = ''; +for await (const chunk of process.stdin) input += chunk; +const { survey, choices } = JSON.parse(input); +process.stdout.write(buildDdiXml(survey, choices)); diff --git a/tests/live/qwacback/conftest.py b/tests/live/qwacback/conftest.py new file mode 100644 index 0000000..7c96d3d --- /dev/null +++ b/tests/live/qwacback/conftest.py @@ -0,0 +1,120 @@ +"""Fixtures for comparing formtransform's DDI with qwacback's. + +`QWACBACK_URL` reuses a running qwacback. Otherwise the fixture starts the +container from docker-compose.yml and removes it afterwards. The tests are +skipped when the qwacback image can't be pulled (it is private; see +docker-compose.yml). +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +import pytest +import requests + +HERE = Path(__file__).parent +REPO_ROOT = HERE.parents[2] +COMPOSE_FILE = HERE / "docker-compose.yml" +BUILD_DDI = HERE / "build_ddi.mjs" +READY_TIMEOUT_S = 120.0 + + +def _wait_for_api(base: str) -> None: + deadline = time.monotonic() + READY_TIMEOUT_S + while time.monotonic() < deadline: + try: + if requests.get(f"{base}/api", timeout=2).status_code == 200: + return + except requests.RequestException: + pass + time.sleep(1) + pytest.fail(f"qwacback /api not ready within {READY_TIMEOUT_S:.0f}s at {base}") + + +@pytest.fixture(scope="session") +def qwacback_url(): + if url := os.environ.get("QWACBACK_URL"): + base = url.rstrip("/") + _wait_for_api(base) + yield base + return + + if not shutil.which("docker"): + pytest.skip("docker not available (set QWACBACK_URL to use a running qwacback)") + + compose = ["docker", "compose", "-p", f"ft-qwacback-{os.getpid()}", "-f", str(COMPOSE_FILE)] + pull = subprocess.run([*compose, "pull", "qwacback"], capture_output=True, text=True) + image_present = ( + subprocess.run( + ["docker", "image", "inspect", os.environ.get("QWACBACK_IMAGE", "ghcr.io/correlaid/qwacback:latest")], + capture_output=True, + ).returncode + == 0 + ) + if pull.returncode != 0 and not image_present: + pytest.skip(f"qwacback image unavailable, see {COMPOSE_FILE.relative_to(REPO_ROOT)}: {pull.stderr.strip()}") + + subprocess.run([*compose, "up", "-d", "--wait", "--wait-timeout", str(int(READY_TIMEOUT_S))], check=True) + base = f"http://127.0.0.1:{os.environ.get('QWACBACK_PORT', '8090')}" + try: + _wait_for_api(base) + yield base + finally: + subprocess.run([*compose, "down", "-v", "--remove-orphans"], check=False) + + +def _post_json(url: str, payload: dict, max_wait_s: float = 90.0) -> requests.Response: + """POST, backing off on 429: qwacback allows guests 10 conversions a minute.""" + waited = 0.0 + while True: + r = requests.post(url, json=payload, timeout=30) + if r.status_code != 429: + return r + pause = min(float(r.headers.get("Retry-After") or 5.0), 10.0) + if waited + pause > max_wait_s: + return r + time.sleep(pause) + waited += pause + + +@pytest.fixture +def qwacback_ddi(qwacback_url): + """DDI XML from qwacback's POST /api/convert/xlsform-to-ddi.""" + + def _convert(survey: list[dict], choices: dict[str, list[dict]]) -> str: + payload = { + "survey": survey, + "choices": [{"list_name": ln, **c} for ln, cs in choices.items() for c in cs], + "settings": {}, + } + r = _post_json(f"{qwacback_url}/api/convert/xlsform-to-ddi", payload) + assert r.status_code == 200, f"qwacback -> HTTP {r.status_code}: {r.text[:500]}" + return r.text + + return _convert + + +@pytest.fixture(scope="session") +def formtransform_ddi(): + """DDI XML from formtransform's buildDdiXml (the built dist/).""" + if not (REPO_ROOT / "dist" / "index.js").exists(): + pytest.fail("dist/index.js missing: run `npm run build` first") + + def _convert(survey: list[dict], choices: dict[str, list[dict]]) -> str: + out = subprocess.run( + ["node", str(BUILD_DDI)], + input=json.dumps({"survey": survey, "choices": choices}), + capture_output=True, + text=True, + check=False, + ) + assert out.returncode == 0, out.stderr + return out.stdout + + return _convert diff --git a/tests/live/qwacback/docker-compose.yml b/tests/live/qwacback/docker-compose.yml new file mode 100644 index 0000000..f1f2825 --- /dev/null +++ b/tests/live/qwacback/docker-compose.yml @@ -0,0 +1,23 @@ +# qwacback alone, for the DDI equivalence test (test_qwacback_equivalence.py). +# Conversion runs in qwacback's Go code, so the schematron-worker isn't needed. +# +# ghcr.io/correlaid/qwacback is private. Log in to ghcr.io, or build it from a +# qwacback checkout and point QWACBACK_IMAGE at it: +# docker build -t qwacback:local ../qwacback +# QWACBACK_IMAGE=qwacback:local npm run test:live -- -k qwacback +services: + qwacback: + image: ${QWACBACK_IMAGE:-ghcr.io/correlaid/qwacback:latest} + ports: + - "${QWACBACK_PORT:-8090}:8080" + environment: + - QWACBACK_SKIP_SEED=1 + - NATS_PORT=4222 + - NATS_TOKEN=${NATS_TOKEN:-changeme} + command: ["./qwacback", "serve", "--http=0.0.0.0:8080"] + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/api"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 10s diff --git a/tests/live/qwacback/test_qwacback_equivalence.py b/tests/live/qwacback/test_qwacback_equivalence.py new file mode 100644 index 0000000..0fc93dd --- /dev/null +++ b/tests/live/qwacback/test_qwacback_equivalence.py @@ -0,0 +1,296 @@ +"""XLSForm → DDI equivalence between formtransform and qwacback (formtransform#14). + +For every answer type qwacback supports (qwacback/internal/examples/examples.go), +the same XLSForm goes through formtransform's `buildDdiXml` and qwacback's +`POST /api/convert/xlsform-to-ddi`, and the DDI shapes must match. + +Ported from survey2ddi's tests/integration/test_conversion_equivalence.py, with +the same cases. qwacback plans to drop its Go converter for this library +(HANDOVER_QWAC.md); until then this is the parity check for that swap. + +Runs with the live suite: `npm run test:live -- -k qwacback`. See +docker-compose.yml for the qwacback image. +""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + +import pytest + +# --- XML shape helpers ----------------------------------------------------- + + +def _tag(el: ET.Element) -> str: + return el.tag.split("}", 1)[-1] if "}" in el.tag else el.tag + + +def _find_data_dscr(xml_str: str) -> ET.Element: + """Return a -equivalent element. + + formtransform always wraps in /. qwacback returns a bare + or when there's only one, or otherwise. We + synthesize a container in the bare case so callers can iterate uniformly. + """ + root = ET.fromstring(xml_str) + if _tag(root) == "dataDscr": + return root + if _tag(root) in ("var", "varGrp"): + container = ET.Element("dataDscr") + container.append(root) + return container + for e in root.iter(): + if _tag(e) == "dataDscr": + return e + raise AssertionError(f"no // root; got <{_tag(root)}>") + + +def _child(el: ET.Element, name: str) -> ET.Element | None: + for c in el: + if _tag(c) == name: + return c + return None + + +def _children(el: ET.Element, name: str) -> list[ET.Element]: + return [c for c in el if _tag(c) == name] + + +def _text(el: ET.Element | None) -> str | None: + if el is None or el.text is None: + return None + return el.text.strip() or None + + +def _var_shape(v: ET.Element) -> dict: + qstn = _child(v, "qstn") + vfmt = _child(v, "varFormat") + concept = _child(v, "concept") + cats = tuple((_text(_child(c, "catValu")), _text(_child(c, "labl"))) for c in _children(v, "catgry")) + return { + "name": v.get("name"), + "ID": v.get("ID"), + "intrvl": v.get("intrvl"), + "nature": v.get("nature"), + "responseDomainType": qstn.get("responseDomainType") if qstn is not None else None, + "preQTxt": _text(_child(qstn, "preQTxt")) if qstn is not None else None, + "qstnLit": _text(_child(qstn, "qstnLit")) if qstn is not None else None, + "varFormat_type": vfmt.get("type") if vfmt is not None else None, + "varFormat_schema": vfmt.get("schema") if vfmt is not None else None, + "concept": _text(concept), + "concept_vocab": concept.get("vocab") if concept is not None else None, + "catgry": cats, + } + + +def _grp_shape(g: ET.Element) -> dict: + return { + "name": g.get("name"), + "ID": g.get("ID"), + "type": g.get("type"), + "var": tuple((g.get("var") or "").split()), + "varGrp_ref": tuple((g.get("varGrp") or "").split()), + "concept": _text(_child(g, "concept")), + "txt": _text(_child(g, "txt")), + } + + +def _shape(xml_str: str) -> tuple[dict, dict]: + dd = _find_data_dscr(xml_str) + vars_ = {v.get("name"): _var_shape(v) for v in _children(dd, "var")} + grps = {g.get("name"): _grp_shape(g) for g in _children(dd, "varGrp")} + return vars_, grps + + +EQUIVALENT_TYPES = [ + pytest.param( + "single_choice", + [ + { + "type": "select_one bildung", + "name": "bildung", + "label": "Bildungsgrad", + "required": "false", + "appearance": None, + } + ], + { + "bildung": [ + {"name": "1", "label": "Kein Abschluss"}, + {"name": "2", "label": "Abitur"}, + {"name": "3", "label": "Hochschulabschluss"}, + ] + }, + id="single_choice", + ), + pytest.param( + "multiple_choice", + [ + { + "type": "select_multiple tage", + "name": "wochenende", + "label": "Wochenendtage", + "required": "false", + "appearance": None, + } + ], + { + "tage": [ + {"name": "sa", "label": "Samstag"}, + {"name": "so", "label": "Sonntag"}, + ] + }, + id="multiple_choice", + ), + pytest.param( + "single_choice_other", + [ + {"type": "select_one quelle", "name": "src", "label": "Source", "required": "false", "appearance": None}, + {"type": "text", "name": "src_other", "label": "Other", "required": "false", "appearance": None}, + ], + { + "quelle": [ + {"name": "a", "label": "A"}, + {"name": "other", "label": "Other"}, + ] + }, + id="single_choice_other", + ), + pytest.param( + "multiple_choice_other", + [ + {"type": "select_multiple dev", "name": "own", "label": "Own", "required": "false", "appearance": None}, + {"type": "text", "name": "own_other", "label": "Other", "required": "false", "appearance": None}, + ], + { + "dev": [ + {"name": "a", "label": "A"}, + {"name": "other", "label": "Other"}, + ] + }, + id="multiple_choice_other", + ), + pytest.param( + "grid", + [ + {"type": "begin_group", "name": "trust", "label": "Trust", "required": "false", "appearance": "table-list"}, + {"type": "select_one s5", "name": "trust_a", "label": "A", "required": "false", "appearance": None}, + {"type": "end_group", "name": None, "label": None, "required": None, "appearance": None}, + ], + { + "s5": [ + {"name": "1", "label": "One"}, + {"name": "2", "label": "Two"}, + ] + }, + id="grid", + ), + pytest.param( + "integer", + [{"type": "integer", "name": "alter", "label": "Alter", "required": "false", "appearance": None}], + {}, + id="integer", + ), + pytest.param( + "decimal", + [{"type": "decimal", "name": "rating", "label": "Rating", "required": "false", "appearance": None}], + {}, + id="decimal", + ), + pytest.param( + "range", + [{"type": "range", "name": "score", "label": "Score", "required": "false", "appearance": None}], + {}, + id="range", + marks=pytest.mark.xfail( + strict=True, + reason="range isn't a registered type: formtransform emits text, qwacback numeric (formtransform#33)", + ), + ), + pytest.param( + "date", + [{"type": "date", "name": "besuch", "label": "Besuchsdatum", "required": "false", "appearance": None}], + {}, + id="date", + ), + pytest.param( + "text", + [{"type": "text", "name": "anmerkung", "label": "Anmerkungen", "required": "false", "appearance": None}], + {}, + id="text", + ), + pytest.param( + "note", + [{"type": "note", "name": "thanks", "label": "Thank you", "required": "false", "appearance": None}], + {}, + id="note", + marks=pytest.mark.xfail( + strict=True, + reason=( + "by design: a note stores no response, so formtransform folds it into / " + "and emits no ; qwacback's Go converter emits one (see HANDOVER_QWAC.md)" + ), + ), + ), + pytest.param( + "calculate", + [{"type": "calculate", "name": "calc", "label": "Calc", "required": "false", "appearance": None}], + {}, + id="calculate", + ), + pytest.param( + "single_choice_long_list", + [ + { + "type": "select_one_from_file iso_3166_1.csv", + "name": "country", + "label": "Country", + "required": "false", + "appearance": None, + } + ], + {}, + id="single_choice_long_list", + ), + pytest.param( + "multiple_choice_long_list", + [ + { + "type": "select_multiple_from_file iso_3166_1.csv", + "name": "visited", + "label": "Visited", + "required": "false", + "appearance": None, + } + ], + {}, + id="multiple_choice_long_list", + ), + pytest.param( + "section", + # Plain begin_group (no table-list appearance) — both converters drop + # the wrapper and emit only the member vars at the top level. + [ + {"type": "begin_group", "name": "section1", "label": "Section 1", "required": "false", "appearance": None}, + {"type": "text", "name": "q1", "label": "Q1", "required": "false", "appearance": None}, + {"type": "end_group", "name": None, "label": None, "required": None, "appearance": None}, + ], + {}, + id="section", + ), +] + + +# Two cases are strict xfails, so a fix on either side shows up as XPASS: +# `range` (formtransform#33) and `note` (intended; qwacback changes on the swap). + + +class TestXlsformToDdiEquivalence: + """formtransform and qwacback produce the same DDI for every supported type.""" + + @pytest.mark.parametrize("label,survey,choices", EQUIVALENT_TYPES) + def test_type_equivalence(self, formtransform_ddi, qwacback_ddi, label, survey, choices): + our_vars, our_grps = _shape(formtransform_ddi(survey, choices)) + their_vars, their_grps = _shape(qwacback_ddi(survey, choices)) + assert our_vars == their_vars, f"{label}: var shape differs" + assert our_grps == their_grps, f"{label}: varGrp shape differs"