From 0007e732879fcc6ca51538c6f13ea4db1e521aaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 16:23:43 +0200 Subject: [PATCH 01/10] Setup Pages Github action. --- .github/workflows/main.yml | 0 .github/workflows/pages.yaml | 32 +++++++++---- .github/workflows/plantuml.yml | 28 ----------- .gitignore | 5 +- pyproject.toml | 4 +- setup.py | 46 +++++++++++++++++++ tools/README.md | 79 ++++++++++++++++++++++++++++---- tools/build.sh | 61 ++++++++++++++++++++++++ tools/configuration.py | 3 +- tools/expand_docs/expand_docs.py | 50 ++++++++++++++------ 10 files changed, 246 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/main.yml delete mode 100644 .github/workflows/plantuml.yml create mode 100644 setup.py create mode 100644 tools/build.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index 01502b13..90be6423 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -1,6 +1,6 @@ -# This is a basic workflow to help you get started with Actions +# Builds the docs and deploys them to GitHub pages. -name: CI +name: Pages # Controls when the workflow will run on: @@ -10,12 +10,17 @@ on: pull_request: branches: [ "main" ] +permissions: + contents: read + pages: write + id-token: write + # Allows you to run this workflow manually from the Actions tab workflow_dispatch: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" + build: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -27,10 +32,19 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo Hello, world! + run: | + ./tools/build.sh + + - uses: actions/upload-pages-artifact@v3 + with: + path: site - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/plantuml.yml b/.github/workflows/plantuml.yml deleted file mode 100644 index 6eaaa3e7..00000000 --- a/.github/workflows/plantuml.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Embed PlantUML in Markdown - -on: - push: - branches: [ main ] - paths: - - '**/*.md' - - '**/*.puml' - pull_request: - paths: - - '**/*.md' - - '**/*.puml' - workflow_dispatch: - -permissions: - contents: write - -jobs: - embed-puml-markdown: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Embed PlantUML inside Markdown - uses: alessandro-marcantoni/puml-markdown@v0.1.1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index b023e8ab..40044042 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,7 @@ generated/xcore/contab/netex.html templates/README.md~ tools/schematron_builder/__pycache__/template2schematron.cpython-313.pyc __pycache__ -netex_rg_ch.egg-info \ No newline at end of file +netex_rg_ch.egg-info +dist/ +site/ +build/ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 82bfc82b..fc75763a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["setuptools>=70", "wheel"] +requires = ["setuptools>=70"] build-backend = "setuptools.build_meta" [project] name = "netex-rg-ch" -version = "0.1.0" +version = "0.1.1" description = "Provides tools to help building the NeTEx RG." requires-python = ">=3.13" dependencies = [ diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..c2be3921 --- /dev/null +++ b/setup.py @@ -0,0 +1,46 @@ +# setup.py - tiny hook to build docs into ./site) +import os +import subprocess +import sys +from pathlib import Path +from setuptools import setup, Command +from setuptools.command.sdist import sdist as _sdist + +class build_docs(Command): + """Generate documentation into ./site.""" + description = "Build project documentation into ./site" + user_options = [ + ("clean", None, "clean the output directory before building"), + ] + + def initialize_options(self): + self.clean = False + + def finalize_options(self): + self.clean = bool(self.clean) + + def run(self): + outdir = Path("site") + if self.clean and outdir.exists(): + import shutil + shutil.rmtree(outdir) + outdir.mkdir(exist_ok=True) + + # run tools here ... + cmd = [sys.executable, "-m", "tools.expand_docs.expand_docs", "--out", str(outdir)] + self.announce(f"Running: {' '.join(cmd)}", level=2) + # Build environment variables if you want to pass context + env = dict(os.environ) + env.setdefault("PYTHONHASHSEED", "0") + subprocess.check_call(cmd, env=env) # fails the build on nonzero exit + self.announce(f"Docs generated in {outdir}", level=2) + +class sdist(_sdist): + def run(self): + # Build docs before sdist (writes to ./site but we will exclude it from sdist) + self.run_command("build_docs") + super().run() + +cmdclass = {"build_docs": build_docs, "sdist": sdist} + +setup(cmdclass=cmdclass) diff --git a/tools/README.md b/tools/README.md index f60e4e79..0b42c5fc 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,16 +1,53 @@ # Tools for the Swiss NeTEx RG -## Install Tools with uv +## How to setup and run the build -The package manager `uv` simplifies the build and installation of scripts for the tools. +The build builds the tools and runs them to create the generated documents in the directory `site`. -- Dependencies are managed by `uv`, as configured in `pyproject.toml` and more detailed in `uv.lock`. -- `uv` provides an os-independent interface for scripts -- The generated tool scripts run on Windows, Mac or Linux +### Steps involved to setup and run the build + +1. Install the [uv package manager](#install-the-uv-package-manager) +2. Initialize the [virtual environment](#initialize-the-virtual-environment) +3. [Run the build]() + +For more information about the build framework, see [Build Automation](#build-automation). + +### Install the uv package manager + +Install the uv package manager: +- See [uv package manager](https://docs.astral.sh/uv/) +- if you have pip installed, you can run `pip install uv` + +### Initialize the virtual environment + +#### Mac/Linux + +Run the following following commands in the project root directory: +```sh +uv venv +uv sync +sh ./venv/bin/activate +``` + +#### Windows + +Run the following following commands in the project root directory: + +``` shell +uv venv +uv sync +venv\bin\activate.bat +``` -### Install the package manager +### Run the build -See [uv package manager](https://docs.astral.sh/uv/) +If everything is setup correctly, you should be able to run the build doing `python -m build` in the project root directory. + + +## Tool Scripts + +The `pyproject.toml` is configured to generate scripts for the tools. +These tool scripts are not required for the build, but they may be useful for running tools locally. ### Prerequisites: Set PYTHONPATH and PATH @@ -52,4 +89,30 @@ This generates executable scripts for Linux/Mac and Windows in subdirectories of ### How to add a new Script - Add a new entry in the `[project.scripts]` section of `pyproject.toml`. -- If the script requires another package, use `uv add` to added to the environment. \ No newline at end of file +- If the script requires another package, use `uv add` to added to the environment. + +## Build Automation Framework + +### Package Manager + +The package manager `uv` simplifies the build and installation of scripts for the tools. + +- Dependencies are managed by `uv`, as configured in `pyproject.toml` and more detailed in `uv.lock`. +- `uv` provides an os-independent interface for scripts + - Generated tool scripts run on Windows, Mac or Linux + +### Project build + +Components of the build automation: +- [pyproject.toml](../pyproject.toml) is configured with `setuptools` (https://setuptools.pypa.io/en/latest/) + - docs can be generated running `python -m build` +- `setup.py` in the root project acts as the interface for the build + - here we can add tools to be run during the build. +- The build writes all output to directory `site`, excluded from git + +### Github Action + +The Github Action [pages.yaml](../.github/pages.yaml) runs the script [build.sh](./build.sh) (can also be tested locally) + - triggered after commits to main branch (e.g. after the merge of a branch) + - runs the build via the `python -m build` mechanism + - uploads generated docs to GitHub Pages diff --git a/tools/build.sh b/tools/build.sh new file mode 100644 index 00000000..b8a27417 --- /dev/null +++ b/tools/build.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Build script for the github action. +# - can also be run and tested locally (Mac or Linux) +# - use "python -m build to build docs otherwise +set -Eeuo pipefail +IFS=$'\n\t' + +# --- Configuration (override via environment variables) --- +PYTHON="${PYTHON_BIN:-python3}" # or "python" if that's your default +VENV_DIR="${VENV_DIR:-.venv-build}" # ephemeral venv only for build tools +OUTDIR="${OUTDIR:-dist}" # where artifacts go +EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:-}" # e.g. "--no-isolation" (not recommended) +TEST_INSTALL="${TEST_INSTALL:-1}" # 1 to smoke-test installing the wheel + +# --- Move to repo root --- +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." &>/dev/null && pwd)" +cd "$REPO_ROOT" + +echo "==> Building package in $REPO_ROOT" + +# --- Pre-flight checks --- +if [[ ! -f pyproject.toml ]]; then + echo "Error: pyproject.toml not found in repository root: $REPO_ROOT" >&2 + exit 1 +fi + +# --- Clean previous build outputs --- +# rm -rf "$OUTDIR" build .pytest_cache +mkdir -p "$OUTDIR" + +# --- Create isolated venv for build tools --- +if [[ ! -d "$VENV_DIR" ]]; then + echo "==> Creating build venv at $VENV_DIR" + $PYTHON -m venv "$VENV_DIR" +fi + +# shellcheck source=/dev/null +source "$VENV_DIR/bin/activate" +$PYTHON -m pip install --upgrade pip setuptools +$PYTHON -m pip install --upgrade build + +# If you use setuptools-scm, ensure it's present for version derivation +if grep -qiE 'setuptools[-_]scm' pyproject.toml; then + $PYTHON -m pip install --upgrade setuptools-scm +fi + +# --- Make builds more reproducible (optional but harmless) --- +export PYTHONHASHSEED=0 +export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct 2>/dev/null || date +%s)}" + +# --- Build sdist and wheel --- +echo "==> Running python -m build" +$PYTHON -m build --sdist --wheel --outdir "$OUTDIR" $EXTRA_BUILD_ARGS + +# --- List outputs --- +echo "==> Produced artifacts:" +ls -lh "$OUTDIR" + +deactivate +echo "==> Build complete." diff --git a/tools/configuration.py b/tools/configuration.py index d780577e..57cc4d12 100644 --- a/tools/configuration.py +++ b/tools/configuration.py @@ -7,6 +7,7 @@ TEMPLATES_DIR = PROJECT_DIR.joinpath("../templates") # Generated documents -GENERATED_DIR = PROJECT_DIR.joinpath("../generated") +GENERATED_DIR = PROJECT_DIR.joinpath("../site") +GENERATED_DOCS_DIR = GENERATED_DIR.joinpath("/docs") XSD_FILE_PATH = PROJECT_DIR.joinpath("../xsd/xsd/NeTEx_publication.xsd") diff --git a/tools/expand_docs/expand_docs.py b/tools/expand_docs/expand_docs.py index 34a3e419..177272e5 100644 --- a/tools/expand_docs/expand_docs.py +++ b/tools/expand_docs/expand_docs.py @@ -6,15 +6,22 @@ import shutil import argparse import re +from abc import ABC + +from setuptools import Command + +from tools.configuration import DOCS_DIR, GENERATED_DOCS_DIR + def copy_media_folder(input_folder, output_folder): """Copy media folder from input to output.""" media_src = os.path.join(input_folder, 'media') media_dst = os.path.join(output_folder, 'media') - + if os.path.exists(media_src): shutil.copytree(media_src, media_dst, dirs_exist_ok=True) + def include_xml_snippet(match, base_folder): """Include XML snippet content directly.""" snippet_path = match.group(1) @@ -23,12 +30,13 @@ def include_xml_snippet(match, base_folder): # base_folder is already the docs folder, so go up one level to project root project_root = os.path.abspath(os.path.join(base_folder, '..')) full_path = os.path.join(project_root, 'generated', 'xml-snippets', snippet_path) - + if os.path.exists(full_path): with open(full_path, 'r', encoding='utf-8') as f: return f"```xml\n{f.read()}\n```" return match.group(0) + def include_markdown_table(match, base_folder): """Include markdown table content directly.""" table_path = match.group(1) @@ -37,7 +45,7 @@ def include_markdown_table(match, base_folder): # base_folder is already the docs folder, so go up one level to project root project_root = os.path.abspath(os.path.join(base_folder, '..')) full_path = os.path.join(project_root, 'generated', 'markdown-examples', table_path) - + if os.path.exists(full_path): with open(full_path, 'r', encoding='utf-8') as f: content = f.read() @@ -54,43 +62,59 @@ def include_markdown_table(match, base_folder): return '\n'.join(table_lines) return match.group(0) + def process_markdown_file(input_path, output_path, base_folder): """Process a single markdown file.""" with open(input_path, 'r', encoding='utf-8') as f: content = f.read() - + # Process XML snippets - match the entire line prefix and link (flexible text) xml_pattern = r'(?:- )?\[.*?\]\(\.\./generated/xml-snippets/([^)]+\.xml)\)' content = re.sub(xml_pattern, lambda m: '\n\n' + include_xml_snippet(m, base_folder) + '\n\n', content) - + # Process markdown tables - match the entire line prefix and link (flexible text) md_pattern = r'(?:- )?\[.*?\]\(\.\./generated/markdown-examples/([^)]+\.md)\)' content = re.sub(md_pattern, lambda m: '\n\n' + include_markdown_table(m, base_folder) + '\n\n', content) - + # Write processed content with open(output_path, 'w', encoding='utf-8') as f: f.write(content) + def main(): parser = argparse.ArgumentParser(description='Expand documentation by including examples and tables.') - parser.add_argument('--docs', required=True, help='Input documentation folder') - parser.add_argument('--out', required=True, help='Output folder') + parser.add_argument('--docs', default=DOCS_DIR, help=f"Input documentation folder (default = {DOCS_DIR})") + parser.add_argument('--out', default=GENERATED_DOCS_DIR, help=f"Output folder (default = {GENERATED_DOCS_DIR})") args = parser.parse_args() - + # Create output folder if it doesn't exist os.makedirs(args.out, exist_ok=True) - + # Copy media folder copy_media_folder(args.docs, args.out) - + # Process each markdown file for filename in os.listdir(args.docs): if filename.endswith('.md'): input_path = os.path.join(args.docs, filename) output_path = os.path.join(args.out, filename) process_markdown_file(input_path, output_path, args.docs) - + print(f"Documentation expanded successfully to {args.out}") + if __name__ == '__main__': - main() \ No newline at end of file + main() + + +class ExpandDocs(Command, ABC): + """Setuptools plugin for the project build.""" + + def run(self) -> None: + """ + Execute the actions intended by the command. + (Side effects **SHOULD** only take place when :meth:`run` is executed, + for example, creating new files or writing to the terminal output). + """ + main() + pass \ No newline at end of file From 7d065d9e200fc2b407249c4552ba95cf4588db08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 16:35:30 +0200 Subject: [PATCH 02/10] Correct description for setup of virtual env. --- tools/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/README.md b/tools/README.md index 0b42c5fc..57239fd7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -24,9 +24,9 @@ Install the uv package manager: Run the following following commands in the project root directory: ```sh -uv venv +uv .venv +source .venv/bin/activate uv sync -sh ./venv/bin/activate ``` #### Windows @@ -34,9 +34,9 @@ sh ./venv/bin/activate Run the following following commands in the project root directory: ``` shell -uv venv +uv .venv +.venv\bin\activate.bat uv sync -venv\bin\activate.bat ``` ### Run the build From bcd50f2d30cb3ad8be895adf3ff3c750fd97d917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 16:36:31 +0200 Subject: [PATCH 03/10] Update uv.lock. --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 09019363..c1ed9cc8 100644 --- a/uv.lock +++ b/uv.lock @@ -217,7 +217,7 @@ wheels = [ [[package]] name = "netex-rg-ch" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "lxml" }, From 2b4997af0cc9838c282acdfdc8a88d9c9bd74491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 17:51:28 +0200 Subject: [PATCH 04/10] Update description of build module installation. --- .gitignore | 3 ++- pyproject.toml | 6 +++--- tools/README.md | 18 ++++++++++++++---- uv.lock | 4 ++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 40044042..6d4a4cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ __pycache__ netex_rg_ch.egg-info dist/ site/ -build/ \ No newline at end of file +build/ +netex_rg_ch_test.egg-info/ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fc75763a..48de31a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=70"] build-backend = "setuptools.build_meta" [project] -name = "netex-rg-ch" -version = "0.1.1" -description = "Provides tools to help building the NeTEx RG." +name = "netex-rg-ch-test" +version = "0.1.2" +description = "Testing tools to help building the NeTEx RG." requires-python = ">=3.13" dependencies = [ "lxml","pyschematron" diff --git a/tools/README.md b/tools/README.md index 57239fd7..ebd55af7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -8,7 +8,8 @@ The build builds the tools and runs them to create the generated documents in th 1. Install the [uv package manager](#install-the-uv-package-manager) 2. Initialize the [virtual environment](#initialize-the-virtual-environment) -3. [Run the build]() +3. Install the [build module](#install-build-module) +4. [Run the build]() For more information about the build framework, see [Build Automation](#build-automation). @@ -24,7 +25,7 @@ Install the uv package manager: Run the following following commands in the project root directory: ```sh -uv .venv +uv venv source .venv/bin/activate uv sync ``` @@ -34,15 +35,24 @@ uv sync Run the following following commands in the project root directory: ``` shell -uv .venv +uv venv .venv\bin\activate.bat uv sync ``` +### Install build module +Make sure you have an up-to-date version of `pip` and of module `build` used to run the build: +``` +python -m ensurepip +python -m pip install --upgrade pip build +``` ### Run the build -If everything is setup correctly, you should be able to run the build doing `python -m build` in the project root directory. +If everything is setup correctly, you should be able to the build from your project root directory: +``` +python -m build +``` ## Tool Scripts diff --git a/uv.lock b/uv.lock index c1ed9cc8..8c6dbab0 100644 --- a/uv.lock +++ b/uv.lock @@ -216,8 +216,8 @@ wheels = [ ] [[package]] -name = "netex-rg-ch" -version = "0.1.1" +name = "netex-rg-ch-test" +version = "0.1.2" source = { editable = "." } dependencies = [ { name = "lxml" }, From a654c12447ae081f5ecf9f9d7913d13284078aab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 18:02:01 +0200 Subject: [PATCH 05/10] Move github action property to right place. --- .github/workflows/pages.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index 90be6423..a4e07fc1 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -10,14 +10,14 @@ on: pull_request: branches: [ "main" ] + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + permissions: contents: read pages: write id-token: write - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: From 710dedb474574c74d9e00c4d8d65a2953573ed31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 22 May 2026 18:06:39 +0200 Subject: [PATCH 06/10] Make build.sh executable. --- tools/build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/build.sh diff --git a/tools/build.sh b/tools/build.sh old mode 100644 new mode 100755 From 77d4fa3479ad16d43035e45f00fc862f15411332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 29 May 2026 13:48:21 +0200 Subject: [PATCH 07/10] Introduce md2html and toolchain.py. --- pyproject.toml | 2 +- setup.py | 21 ++++----- tools/configuration.py | 2 +- tools/expand_docs/expand_docs.py | 41 ++++++------------ tools/md2html/__init__.py | 0 tools/md2html/md2html.py | 74 ++++++++++++++++++++++++++++++++ tools/toolchain.py | 17 ++++++++ uv.lock | 13 ++++++ 8 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 tools/md2html/__init__.py create mode 100644 tools/md2html/md2html.py create mode 100644 tools/toolchain.py diff --git a/pyproject.toml b/pyproject.toml index 48de31a2..611e2783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.1.2" description = "Testing tools to help building the NeTEx RG." requires-python = ">=3.13" dependencies = [ - "lxml","pyschematron" + "lxml","pyschematron","markdown","pygments" ] [project.scripts] diff --git a/setup.py b/setup.py index c2be3921..44b24324 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,13 @@ import os import subprocess import sys -from pathlib import Path + from setuptools import setup, Command from setuptools.command.sdist import sdist as _sdist class build_docs(Command): - """Generate documentation into ./site.""" - description = "Build project documentation into ./site" + + description = "Build project documentation." user_options = [ ("clean", None, "clean the output directory before building"), ] @@ -20,20 +20,15 @@ def finalize_options(self): self.clean = bool(self.clean) def run(self): - outdir = Path("site") - if self.clean and outdir.exists(): - import shutil - shutil.rmtree(outdir) - outdir.mkdir(exist_ok=True) - - # run tools here ... - cmd = [sys.executable, "-m", "tools.expand_docs.expand_docs", "--out", str(outdir)] + + cmd = [sys.executable, "-m", "tools.toolchain", "--target", "docs"] + self.announce(f"Running: {' '.join(cmd)}", level=2) - # Build environment variables if you want to pass context + # Build environment variables to pass context env = dict(os.environ) env.setdefault("PYTHONHASHSEED", "0") subprocess.check_call(cmd, env=env) # fails the build on nonzero exit - self.announce(f"Docs generated in {outdir}", level=2) + self.announce(f"Docs generated.", level=2) class sdist(_sdist): def run(self): diff --git a/tools/configuration.py b/tools/configuration.py index 57cc4d12..21419320 100644 --- a/tools/configuration.py +++ b/tools/configuration.py @@ -8,6 +8,6 @@ # Generated documents GENERATED_DIR = PROJECT_DIR.joinpath("../site") -GENERATED_DOCS_DIR = GENERATED_DIR.joinpath("/docs") +GENERATED_DOCS_DIR = GENERATED_DIR.joinpath("docs") XSD_FILE_PATH = PROJECT_DIR.joinpath("../xsd/xsd/NeTEx_publication.xsd") diff --git a/tools/expand_docs/expand_docs.py b/tools/expand_docs/expand_docs.py index 177272e5..f5b2758f 100644 --- a/tools/expand_docs/expand_docs.py +++ b/tools/expand_docs/expand_docs.py @@ -80,41 +80,28 @@ def process_markdown_file(input_path, output_path, base_folder): with open(output_path, 'w', encoding='utf-8') as f: f.write(content) - -def main(): - parser = argparse.ArgumentParser(description='Expand documentation by including examples and tables.') - parser.add_argument('--docs', default=DOCS_DIR, help=f"Input documentation folder (default = {DOCS_DIR})") - parser.add_argument('--out', default=GENERATED_DOCS_DIR, help=f"Output folder (default = {GENERATED_DOCS_DIR})") - args = parser.parse_args() - +def expand_docs(input_dir: str, output_dir: str): # Create output folder if it doesn't exist - os.makedirs(args.out, exist_ok=True) + os.makedirs(output_dir, exist_ok=True) # Copy media folder - copy_media_folder(args.docs, args.out) + copy_media_folder(input_dir, output_dir) # Process each markdown file - for filename in os.listdir(args.docs): + for filename in os.listdir(input_dir): if filename.endswith('.md'): - input_path = os.path.join(args.docs, filename) - output_path = os.path.join(args.out, filename) - process_markdown_file(input_path, output_path, args.docs) + input_md = os.path.join(input_dir, filename) + output_md = os.path.join(output_dir, filename) + process_markdown_file(input_md, output_md, input_dir) - print(f"Documentation expanded successfully to {args.out}") + print(f"Documentation expanded successfully to {output_dir}") +def main(): + parser = argparse.ArgumentParser(description='Expand documentation by including examples and tables.') + parser.add_argument('--docs', default=DOCS_DIR, help=f"Input documentation folder (default = {DOCS_DIR})") + parser.add_argument('--out', default=GENERATED_DOCS_DIR, help=f"Output folder (default = {GENERATED_DOCS_DIR})") + args = parser.parse_args() + expand_docs(args.docs, args.out) if __name__ == '__main__': main() - - -class ExpandDocs(Command, ABC): - """Setuptools plugin for the project build.""" - - def run(self) -> None: - """ - Execute the actions intended by the command. - (Side effects **SHOULD** only take place when :meth:`run` is executed, - for example, creating new files or writing to the terminal output). - """ - main() - pass \ No newline at end of file diff --git a/tools/md2html/__init__.py b/tools/md2html/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/md2html/md2html.py b/tools/md2html/md2html.py new file mode 100644 index 00000000..5c5c78da --- /dev/null +++ b/tools/md2html/md2html.py @@ -0,0 +1,74 @@ +import argparse +import os +import re + +import markdown +from pygments.formatters import HtmlFormatter + +from tools.configuration import GENERATED_DOCS_DIR + +MD_LINK_PATTERN = re.compile(r'(\[.*\])(\(.*\.)(md)(.*\))') +MD_LINK_REPLACEMENT = r'\1\2html\4' + +def generate_html(md: str) -> str: + + md_with_html_links = MD_LINK_PATTERN.sub(MD_LINK_REPLACEMENT, md) + body = generate_html_body(md_with_html_links) + + # Generate CSS for code highlighting + code_css = HtmlFormatter().get_style_defs(".codehilite") + + # Wrap in a simple HTML template + return f""" + + + + Title + + + + + {body} + + """ + +def generate_html_body(md: str) -> str: + """Convert Markdown to HTML fragment.""" + body = markdown.markdown( + md, + extensions=[ + "extra", # tables, footnotes, etc. + "fenced_code", # triple-backtick code blocks + "codehilite" # Pygments-based syntax highlighting + ], + extension_configs={ + "codehilite": {"guess_lang": True, "noclasses": False} + }, + ) + return body + +def generate_html_files(src_dir: str): + """Generate HTML files from all markdown files in src_dir.""" + for filename in os.listdir(src_dir): + if filename.endswith('.md'): + md_path = os.path.join(src_dir, filename) + with open(md_path, 'r', encoding='utf-8') as f: + md = f.read() + html = generate_html(md) + html_path = md_path.replace('.md', '.html') + with open(html_path, 'w', encoding='utf-8') as f: + f.write(html) + print(f"Generated HTML files in {src_dir}") + +def main(): + parser = argparse.ArgumentParser(description='Generate HTML files from Markdown files.') + parser.add_argument('--dir', default=GENERATED_DOCS_DIR, help=f"Folder to search for md files (default = {GENERATED_DOCS_DIR})") + args = parser.parse_args() + generate_html_files(args.dir) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/tools/toolchain.py b/tools/toolchain.py new file mode 100644 index 00000000..e3f4dc35 --- /dev/null +++ b/tools/toolchain.py @@ -0,0 +1,17 @@ +import argparse +import shutil + +from tools.configuration import DOCS_DIR, GENERATED_DOCS_DIR +from tools.expand_docs.expand_docs import expand_docs +from tools.md2html.md2html import generate_html_files + + +def main(): + parser = argparse.ArgumentParser(description='Run tools to generate documents.') + parser.add_argument('--target', default="docs", help=f"Target to generate (default: docs)") + args = parser.parse_args() + expand_docs(DOCS_DIR, GENERATED_DOCS_DIR) + generate_html_files(GENERATED_DOCS_DIR) + +if __name__ == '__main__': + main() diff --git a/uv.lock b/uv.lock index 8c6dbab0..6699d47b 100644 --- a/uv.lock +++ b/uv.lock @@ -142,6 +142,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/39/53/8ba3cd5984f8363635450c93f63e541a0721b362bb32ae0d8237d9674aee/lxml-6.0.4-cp314-cp314t-win_arm64.whl", hash = "sha256:1dcd9e6cb9b7df808ea33daebd1801f37a8f50e8c075013ed2a2343246727838", size = 3816184, upload-time = "2026-04-12T16:26:57.011Z" }, ] +[[package]] +name = "markdown" +version = "3.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", size = 368805, upload-time = "2026-02-09T14:57:26.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", size = 108180, upload-time = "2026-02-09T14:57:25.787Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -221,12 +230,16 @@ version = "0.1.2" source = { editable = "." } dependencies = [ { name = "lxml" }, + { name = "markdown" }, + { name = "pygments" }, { name = "pyschematron" }, ] [package.metadata] requires-dist = [ { name = "lxml" }, + { name = "markdown" }, + { name = "pygments" }, { name = "pyschematron" }, ] From 7e5c50a2fb99da0357d11997b5d505d6509b4c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 29 May 2026 14:09:05 +0200 Subject: [PATCH 08/10] Add dependencies for the build. --- pyproject.toml | 4 ++-- uv.lock | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 611e2783..14a83c40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=70"] +requires = ["setuptools>=70","markdown","pygments"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +8,7 @@ version = "0.1.2" description = "Testing tools to help building the NeTEx RG." requires-python = ">=3.13" dependencies = [ - "lxml","pyschematron","markdown","pygments" + "lxml","pyschematron","pygments","build","pip","markdown" ] [project.scripts] diff --git a/uv.lock b/uv.lock index 6699d47b..a9382746 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, ] +[[package]] +name = "build" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "os_name == 'nt'" }, + { name = "packaging" }, + { name = "pyproject-hooks" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/e0/df5e171f685f82f37b12e1f208064e24244911079d7b767447d1af7e0d70/build-1.5.0.tar.gz", hash = "sha256:302c22c3ba2a0fd5f3911918651341ebb3896176cbdec15bd421f80b1afc7647", size = 89796, upload-time = "2026-04-30T03:18:25.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/fe/6bea5c9162869c5beba5d9c8abbed835ec85bf1ec1fba05a3822325c45f3/build-1.5.0-py3-none-any.whl", hash = "sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f", size = 26018, upload-time = "2026-04-30T03:18:23.644Z" }, +] + [[package]] name = "click" version = "8.3.3" @@ -229,20 +243,42 @@ name = "netex-rg-ch-test" version = "0.1.2" source = { editable = "." } dependencies = [ + { name = "build" }, { name = "lxml" }, { name = "markdown" }, + { name = "pip" }, { name = "pygments" }, { name = "pyschematron" }, ] [package.metadata] requires-dist = [ + { name = "build" }, { name = "lxml" }, { name = "markdown" }, + { name = "pip" }, { name = "pygments" }, { name = "pyschematron" }, ] +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pip" +version = "26.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/48/cb9b7a682f6fe01a4221e1728941dd4ac3cd9090a17db3779d6ff490b602/pip-26.1.1.tar.gz", hash = "sha256:d36762751d156a4ee895de8af39aa0abeeeb577f93a2eca6ab62467bbf0f8a78", size = 1840400, upload-time = "2026-05-04T19:02:21.248Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/eb/fea4d1d51c49832120f7f285d07306db3960f423a2612c6057caf3e8196f/pip-26.1.1-py3-none-any.whl", hash = "sha256:99cb1c2899893b075ff56e4ed0af55669a955b49ad7fb8d8603ecdaf4ed653fb", size = 1812777, upload-time = "2026-05-04T19:02:18.9Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -252,6 +288,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyproject-hooks" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, +] + [[package]] name = "pyschematron" version = "1.1.14" From 6d8f890ddaf7b29628e008f7e444ddcc671c38f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 29 May 2026 14:14:09 +0200 Subject: [PATCH 09/10] Optimize imports. --- tools/expand_docs/expand_docs.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/expand_docs/expand_docs.py b/tools/expand_docs/expand_docs.py index f5b2758f..03333902 100644 --- a/tools/expand_docs/expand_docs.py +++ b/tools/expand_docs/expand_docs.py @@ -6,13 +6,8 @@ import shutil import argparse import re -from abc import ABC - -from setuptools import Command - from tools.configuration import DOCS_DIR, GENERATED_DOCS_DIR - def copy_media_folder(input_folder, output_folder): """Copy media folder from input to output.""" media_src = os.path.join(input_folder, 'media') From 1ba8c95d1adb583f4ce6bebfe133d3e068f994cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Urs=20St=C3=B6ckli?= Date: Fri, 29 May 2026 14:21:54 +0200 Subject: [PATCH 10/10] Documentation note regarding build module. --- tools/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/README.md b/tools/README.md index ebd55af7..be6e7f60 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,6 +41,8 @@ uv sync ``` ### Install build module +> This can be skipped as the `build` module is now part of the build dependencies. + Make sure you have an up-to-date version of `pip` and of module `build` used to run the build: ``` python -m ensurepip