spec conversion

.github/workflows/spec.yml

@@ -0,0 +1,91 @@
+name: spec
+
+# Builds the DFDL specification and runs the checks. Everything that renders
+# the spec happens inside the pinned Metanorma image the Makefile names, so
+# the runner installs nothing beyond ruff and black.
+#
+# Run it from the Actions tab. There is deliberately no push or pull_request
+# trigger: whether this repository runs CI automatically, and on what, is a
+# working group decision rather than something this workflow assumes. To make
+# it automatic, add the triggers you want:
+#
+#   on:
+#     workflow_dispatch:
+#     pull_request:
+#     push:
+#       branches: [master]
+
+on:
+  workflow_dispatch:
+
+# Read-only: this workflow never writes to the repository, publishes nothing,
+# and uses no secrets, so it behaves identically for pull requests from forks.
+permissions:
+  contents: read
+
+# A newer run for the same ref supersedes an older one.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  # Kept in step with IMAGE in the Makefile.
+  METANORMA_IMAGE: metanorma/metanorma:alpine-1.17.0
+
+jobs:
+  build:
+    name: build (both editions)
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Pull the pinned Metanorma image
+        run: docker pull "$METANORMA_IMAGE"
+
+      # Both editions, because a change can break one and not the other.
+      - name: Build the specification
+        run: make editions
+
+      - name: List the build output
+        run: ls -lR build
+
+      # The artifacts are how a reviewer reads the rendered spec without
+      # installing Docker, Metanorma or anything else.
+      - name: Upload the rendered specification
+        uses: actions/upload-artifact@v4
+        with:
+          name: dfdl-spec
+          path: |
+            build/iso/dfdl.pdf
+            build/iso/dfdl.html
+            build/ogf/dfdl.pdf
+            build/ogf/dfdl.html
+          if-no-files-found: error
+
+  check:
+    name: check (lint + validators)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          # Matches target-version in pyproject.toml.
+          python-version: '3.14'
+
+      - name: Install the linters
+        run: python -m pip install --disable-pip-version-check ruff==0.16.5 black==26.5.1
+
+      - name: Pull the pinned Metanorma image
+        run: docker pull "$METANORMA_IMAGE"
+
+      # `make check` runs ruff and black over tools/, builds the document,
+      # then runs each validator in tools/ against it. Any violation exits
+      # non-zero and fails this job.
+      - name: Lint and check fidelity
+        run: make check

.gitignore

@@ -1 +1,4 @@
-/_site/
+/_site/
+build/
+relaton/
+__pycache__/

Makefile

@@ -0,0 +1,123 @@
+# DFDL specification build.
+#
+# Metanorma runs inside a pinned Docker image, so nothing needs to be
+# installed locally and every build is reproducible.
+#
+#   make xml       semantic XML only (fast)
+#   make html      HTML
+#   make pdf       ISO-formatted PDF (slow)
+#   make all       pdf + html
+#   make editions  `all` for every edition, from one source state
+#   make lint      ruff + black over tools/ (host tools, not the container)
+#   make check     lint, then the validators in tools/ against the semantic XML
+#   make clean     remove build output
+#
+#   make release VERSION=x.y.z
+#                  every edition, rendered from scratch into docs/releases/,
+#                  which is committed; build/ is not.
+#
+# EDITION selects which edition to build; output lands in build/$(EDITION)/.
+# Both editions render identically-named artifacts, so they need separate
+# directories.
+#
+#   make EDITION=ogf pdf
+#
+# ISO's house font (Cambria) is proprietary; --continue-without-fonts lets
+# the build fall back to Noto Sans instead of prompting for a licence.
+#
+# The build also writes a relaton/ bibliography cache at the repo root; it is
+# gitignored and kept between builds so references resolve offline.
+
+IMAGE := metanorma/metanorma:alpine-1.17.0
+STEM  := dfdl
+
+EDITIONS := iso ogf
+EDITION  ?= iso
+SPEC_iso := spec/dfdl.adoc
+SPEC_ogf := spec/dfdl-ogf.adoc
+SPEC     := $(SPEC_$(EDITION))
+BUILD    := build/$(EDITION)
+
+# Metanorma names its output after the input file; the artifacts are
+# renamed to $(STEM) so both editions land under the same names.
+SRCSTEM  := $(basename $(notdir $(SPEC)))
+
+DOCKER   := docker run --rm -v "$(CURDIR):/metanorma" $(IMAGE)
+MN_FLAGS := --no-install-fonts --continue-without-fonts
+
+RELEASE  := docs/releases/$(VERSION)
+
+.PHONY: all editions release xml html pdf lint check check-conversion clean
+
+# One pdf pass also emits the HTML and the XML, so `all` is just `pdf`.
+all: pdf
+
+# Every edition in one invocation, so both come from the same source state.
+editions:
+	@for e in $(EDITIONS); do \
+		echo "==> $$e"; \
+		$(MAKE) --no-print-directory EDITION=$$e all || exit 1; \
+	done
+
+xml:  FORMATS := xml
+html: FORMATS := xml,html
+pdf:  FORMATS := xml,html,pdf
+
+# Metanorma writes its output next to the input file, so move the artifacts
+# into $(BUILD)/ afterwards.
+xml html pdf:
+	@mkdir -p $(BUILD)
+	$(DOCKER) metanorma compile -t iso -x $(FORMATS) $(MN_FLAGS) $(SPEC)
+	@for f in $(dir $(SPEC))$(SRCSTEM).*; do \
+		case "$$f" in *.adoc) continue ;; esac; \
+		mv -f "$$f" "$(BUILD)/$(STEM)$${f#$(dir $(SPEC))$(SRCSTEM)}"; \
+	done
+	@echo "Output in $(BUILD)/"
+
+# A release is committed, so it is rendered from scratch rather than
+# published from whatever happens to be left in build/. The edition is part
+# of the filename because build/ keeps them apart by directory instead.
+release:
+	@test -n "$(VERSION)" || { echo "usage: make release VERSION=x.y.z" >&2; exit 1; }
+	$(MAKE) --no-print-directory clean
+	$(MAKE) --no-print-directory editions
+	@mkdir -p $(RELEASE)
+	@for e in $(EDITIONS); do \
+		for x in pdf html xml; do \
+			cp build/$$e/$(STEM).$$x $(RELEASE)/$(STEM)-$$e.$$x || exit 1; \
+		done; \
+	done
+	@echo "Release in $(RELEASE)/"
+
+# ruff and black run on the host, not in the Metanorma container.
+lint:
+	@if ls tools/*.py >/dev/null 2>&1; then \
+		ruff check tools/ && black --check tools/; \
+	else \
+		echo "No Python in tools/ yet; nothing to lint."; \
+	fi
+
+# Runs every executable in tools/, plus any tools/check*.py, over the
+# build. Tolerant of tools/ being empty or absent.
+check: lint html
+	@checkers=$$({ find tools -maxdepth 1 -type f -perm -u+x; \
+		ls tools/check*.py; } 2>/dev/null | sort -u); \
+	if [ -z "$$checkers" ]; then \
+		echo "No validators in tools/ yet; nothing to check."; \
+	else \
+		for c in $$checkers; do \
+			echo "==> $$c"; \
+			if [ -x "$$c" ]; then "$$c" $(BUILD)/$(STEM).xml; \
+			else python3 "$$c" $(BUILD)/$(STEM).xml; fi || exit 1; \
+		done; \
+	fi
+
+# Compares the build against the Word source, character for character. This
+# proves the conversion; it is not part of `make check`, because a deliberate
+# change to the specification is supposed to differ from GFD.240 and would
+# fail it correctly. See spec/BUILD.md.
+check-conversion: xml
+	python3 tools/fidelity-check.py $(BUILD)/$(STEM).xml
+
+clean:
+	rm -rf build .ruff_cache

README.md

@@ -24,6 +24,8 @@ This working group maintains this standard and proposed extensions/additions to
 * [DFDL Workgroup News](https://github.com/OpenGridForum/DFDL/wiki/DFDL-Workgroup-News)
 * [DFDL Workgroup Calls/Meeting Minutes](https://github.com/OpenGridForum/DFDL/tree/master/calls)
 * [DFDL Workgroup Current Documents](https://github.com/OpenGridForum/DFDL/tree/master/docs/current)
+* [Building the specification](spec/BUILD.md)
+* [Editing the specification](spec/EDITING.md)
 * [OGF DFDL Homepage](http://www.ogf.org/dfdl)
 * [Wikipedia DFDL](https://en.wikipedia.org/wiki/Data_Format_Description_Language)
 

pyproject.toml

@@ -0,0 +1,5 @@
+# Config only for the helper scripts in tools/. This is not a Python package.
+
+[tool.ruff]
+target-version = "py314"
+line-length = 88

spec/BUILD.md

@@ -0,0 +1,116 @@
+# Building the specification
+
+Everything is driven from the `Makefile` at the repository root. `TOOLCHAIN.md`
+records the ISO constraints the toolchain has to meet; this records how to run
+it.
+
+## What you need
+
+Docker, and nothing else, to render the specification: Metanorma runs inside a
+container. `make lint` and `make check` additionally use `ruff`, `black` and
+Python 3 on the host, because the validators in `tools/` are not part of the
+Metanorma image.
+
+## Why the image is pinned
+
+`IMAGE := metanorma/metanorma:alpine-1.17.0`
+
+Metanorma bundles the ISO stylesheets, the XSL-FO renderer and the Relaton
+bibliography stack, and their output moves between releases: page breaks shift,
+generated numbering changes, and new style warnings appear. Pinning the tag
+means a contributor, a reviewer and CI all render the same bytes from the same
+source. The same tag is repeated in `.github/workflows/spec.yml`; change both
+together.
+
+## Targets
+
+| Target | What it does |
+| --- | --- |
+| `make xml` | semantic XML only — the fast path, and what the validators read |
+| `make html` | XML + HTML |
+| `make pdf` | XML + HTML + PDF (slow) |
+| `make all` | the same as `make pdf` — one pass emits all three |
+| `make editions` | `make all` for every edition |
+| `make lint` | `ruff` and `black` over `tools/` |
+| `make check` | `lint`, then every validator in `tools/` against the semantic XML |
+| `make release VERSION=x.y.z` | versioned deliverables for both editions |
+| `make clean` | remove the build output |
+
+`EDITION` selects a single edition, for example `make EDITION=ogf pdf`. It
+defaults to `iso`.
+
+## The validators
+
+`make check` runs everything in `tools/` against the build.
+
+| | |
+|---|---|
+| `link-check.py` | every cross-reference and citation resolves to exactly one target |
+| `structure-check.py` | the rendered page says what it means: no sentence naming the wrong figure, no caption stranded, no markup leaking, nothing published that the Word source hides |
+| `fidelity-check.py` | the converted text matches the Word source, character for character |
+
+The first two answer questions that stay live for as long as the document is
+edited: an erratum can strand a caption or break a reference, and neither shows
+up in a diff a reviewer would notice.
+
+`fidelity-check.py` is not one of them, and is not run by `make check`. It
+compares the build against the Word source character for character, which proves
+the conversion but fails on any deliberate change to the specification: an
+erratum is *supposed* to differ from GFD.240. Run it on its own:
+
+    make check-conversion
+
+Never teach it to ignore differences. A fidelity check that has been relaxed is
+worse than none, because it still reports PASS.
+
+It keeps a second use. "What has changed since GFD.240?" is a question an ISO
+submission has to answer, and this answers it clause by clause.
+
+## The two editions
+
+The specification is published twice: as ISO/IEC 23415 and as OGF GFD.240. The
+normative text is identical, so `spec/body.adoc` holds all of it and is included
+by both masters — `spec/dfdl.adoc` and `spec/dfdl-ogf.adoc` — which differ only
+in document metadata and the front matter they pull in. Editing the body edits
+both editions.
+
+Metanorma names its output after its input, and the Makefile renames the
+artifacts to a common stem, so the two editions produce identically named files.
+They are written to `build/iso/` and `build/ogf/` so neither can silently
+overwrite the other. `make editions` renders both in one invocation, which also
+guarantees they came from the same source state.
+
+Each build directory holds `dfdl.xml` (semantic XML), `dfdl.presentation.xml`,
+`dfdl.html`, `dfdl.pdf`, and the diagnostics: `dfdl.err.html` lists Metanorma's
+style and structure warnings and is worth reading when output looks wrong.
+
+## Releases
+
+`build/` is gitignored, because a rendered PDF in every commit would bury the
+source changes. Versioned deliverables are committed instead, under
+`docs/releases/<VERSION>/`, the same way `docs/current/` holds the Word-era
+`.docx`, `.pdf` and `.htm`.
+
+    make release VERSION=1.2.3
+
+The target refuses to run without `VERSION`, cleans, builds both editions from
+scratch rather than publishing whatever is left in `build/`, and copies the
+deliverables out under names that carry the edition:
+
+    docs/releases/1.2.3/dfdl-iso.pdf   dfdl-iso.html   dfdl-iso.xml
+    docs/releases/1.2.3/dfdl-ogf.pdf   dfdl-ogf.html   dfdl-ogf.xml
+
+## Known toolchain issues
+
+**Cambria is proprietary.** ISO's house font cannot be redistributed, so the
+build passes `--no-install-fonts --continue-without-fonts` and falls back to
+Noto Sans. Font warnings in the log are expected, and line and page breaks in
+the local PDF will not match ISO's own typesetting.
+
+**Revision marks need the macro form.** The AsciiDoc role syntax `[.add]#text#`
+is silently ignored — no error, no mark, the text just renders plain. Only
+`add:[text]` and `del:[text]` produce revision marks.
+
+**The bibliography cache is kept.** Builds write a `relaton/` cache at the
+repository root. It is gitignored and reused, so references resolve without
+network access after the first build; delete it if a reference looks stale.