diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index 01502b13..a4e07fc1 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -1,6 +1,6 @@ -# This is a basic workflow to help you get started with Actions +# Builds the docs and deploys them to GitHub pages. -name: CI +name: Pages # Controls when the workflow will run on: @@ -13,9 +13,14 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: +permissions: + contents: read + pages: write + id-token: write + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" + build: # The type of runner that the job will run on runs-on: ubuntu-latest @@ -27,10 +32,19 @@ jobs: # Runs a single command using the runners shell - name: Run a one-line script - run: echo Hello, world! + run: | + ./tools/build.sh + + - uses: actions/upload-pages-artifact@v3 + with: + path: site - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 \ No newline at end of file diff --git a/.github/workflows/plantuml.yml b/.github/workflows/plantuml.yml deleted file mode 100644 index 6eaaa3e7..00000000 --- a/.github/workflows/plantuml.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Embed PlantUML in Markdown - -on: - push: - branches: [ main ] - paths: - - '**/*.md' - - '**/*.puml' - pull_request: - paths: - - '**/*.md' - - '**/*.puml' - workflow_dispatch: - -permissions: - contents: write - -jobs: - embed-puml-markdown: - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Embed PlantUML inside Markdown - uses: alessandro-marcantoni/puml-markdown@v0.1.1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index b023e8ab..6d4a4cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,8 @@ generated/xcore/contab/netex.html templates/README.md~ tools/schematron_builder/__pycache__/template2schematron.cpython-313.pyc __pycache__ -netex_rg_ch.egg-info \ No newline at end of file +netex_rg_ch.egg-info +dist/ +site/ +build/ +netex_rg_ch_test.egg-info/ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 82bfc82b..14a83c40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,14 @@ [build-system] -requires = ["setuptools>=70", "wheel"] +requires = ["setuptools>=70","markdown","pygments"] build-backend = "setuptools.build_meta" [project] -name = "netex-rg-ch" -version = "0.1.0" -description = "Provides tools to help building the NeTEx RG." +name = "netex-rg-ch-test" +version = "0.1.2" +description = "Testing tools to help building the NeTEx RG." requires-python = ">=3.13" dependencies = [ - "lxml","pyschematron" + "lxml","pyschematron","pygments","build","pip","markdown" ] [project.scripts] diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..44b24324 --- /dev/null +++ b/setup.py @@ -0,0 +1,41 @@ +# setup.py - tiny hook to build docs into ./site) +import os +import subprocess +import sys + +from setuptools import setup, Command +from setuptools.command.sdist import sdist as _sdist + +class build_docs(Command): + + description = "Build project documentation." + user_options = [ + ("clean", None, "clean the output directory before building"), + ] + + def initialize_options(self): + self.clean = False + + def finalize_options(self): + self.clean = bool(self.clean) + + def run(self): + + cmd = [sys.executable, "-m", "tools.toolchain", "--target", "docs"] + + self.announce(f"Running: {' '.join(cmd)}", level=2) + # Build environment variables to pass context + env = dict(os.environ) + env.setdefault("PYTHONHASHSEED", "0") + subprocess.check_call(cmd, env=env) # fails the build on nonzero exit + self.announce(f"Docs generated.", level=2) + +class sdist(_sdist): + def run(self): + # Build docs before sdist (writes to ./site but we will exclude it from sdist) + self.run_command("build_docs") + super().run() + +cmdclass = {"build_docs": build_docs, "sdist": sdist} + +setup(cmdclass=cmdclass) diff --git a/tools/README.md b/tools/README.md index f60e4e79..be6e7f60 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,16 +1,65 @@ # Tools for the Swiss NeTEx RG -## Install Tools with uv +## How to setup and run the build -The package manager `uv` simplifies the build and installation of scripts for the tools. +The build builds the tools and runs them to create the generated documents in the directory `site`. -- Dependencies are managed by `uv`, as configured in `pyproject.toml` and more detailed in `uv.lock`. -- `uv` provides an os-independent interface for scripts -- The generated tool scripts run on Windows, Mac or Linux +### Steps involved to setup and run the build + +1. Install the [uv package manager](#install-the-uv-package-manager) +2. Initialize the [virtual environment](#initialize-the-virtual-environment) +3. Install the [build module](#install-build-module) +4. [Run the build]() + +For more information about the build framework, see [Build Automation](#build-automation). + +### Install the uv package manager + +Install the uv package manager: +- See [uv package manager](https://docs.astral.sh/uv/) +- if you have pip installed, you can run `pip install uv` + +### Initialize the virtual environment + +#### Mac/Linux + +Run the following following commands in the project root directory: +```sh +uv venv +source .venv/bin/activate +uv sync +``` + +#### Windows + +Run the following following commands in the project root directory: + +``` shell +uv venv +.venv\bin\activate.bat +uv sync +``` +### Install build module + +> This can be skipped as the `build` module is now part of the build dependencies. + +Make sure you have an up-to-date version of `pip` and of module `build` used to run the build: +``` +python -m ensurepip +python -m pip install --upgrade pip build +``` +### Run the build + +If everything is setup correctly, you should be able to the build from your project root directory: -### Install the package manager +``` +python -m build +``` -See [uv package manager](https://docs.astral.sh/uv/) +## Tool Scripts + +The `pyproject.toml` is configured to generate scripts for the tools. +These tool scripts are not required for the build, but they may be useful for running tools locally. ### Prerequisites: Set PYTHONPATH and PATH @@ -52,4 +101,30 @@ This generates executable scripts for Linux/Mac and Windows in subdirectories of ### How to add a new Script - Add a new entry in the `[project.scripts]` section of `pyproject.toml`. -- If the script requires another package, use `uv add` to added to the environment. \ No newline at end of file +- If the script requires another package, use `uv add` to added to the environment. + +## Build Automation Framework + +### Package Manager + +The package manager `uv` simplifies the build and installation of scripts for the tools. + +- Dependencies are managed by `uv`, as configured in `pyproject.toml` and more detailed in `uv.lock`. +- `uv` provides an os-independent interface for scripts + - Generated tool scripts run on Windows, Mac or Linux + +### Project build + +Components of the build automation: +- [pyproject.toml](../pyproject.toml) is configured with `setuptools` (https://setuptools.pypa.io/en/latest/) + - docs can be generated running `python -m build` +- `setup.py` in the root project acts as the interface for the build + - here we can add tools to be run during the build. +- The build writes all output to directory `site`, excluded from git + +### Github Action + +The Github Action [pages.yaml](../.github/pages.yaml) runs the script [build.sh](./build.sh) (can also be tested locally) + - triggered after commits to main branch (e.g. after the merge of a branch) + - runs the build via the `python -m build` mechanism + - uploads generated docs to GitHub Pages diff --git a/tools/build.sh b/tools/build.sh new file mode 100755 index 00000000..b8a27417 --- /dev/null +++ b/tools/build.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Build script for the github action. +# - can also be run and tested locally (Mac or Linux) +# - use "python -m build to build docs otherwise +set -Eeuo pipefail +IFS=$'\n\t' + +# --- Configuration (override via environment variables) --- +PYTHON="${PYTHON_BIN:-python3}" # or "python" if that's your default +VENV_DIR="${VENV_DIR:-.venv-build}" # ephemeral venv only for build tools +OUTDIR="${OUTDIR:-dist}" # where artifacts go +EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:-}" # e.g. "--no-isolation" (not recommended) +TEST_INSTALL="${TEST_INSTALL:-1}" # 1 to smoke-test installing the wheel + +# --- Move to repo root --- +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "$SCRIPT_DIR/.." &>/dev/null && pwd)" +cd "$REPO_ROOT" + +echo "==> Building package in $REPO_ROOT" + +# --- Pre-flight checks --- +if [[ ! -f pyproject.toml ]]; then + echo "Error: pyproject.toml not found in repository root: $REPO_ROOT" >&2 + exit 1 +fi + +# --- Clean previous build outputs --- +# rm -rf "$OUTDIR" build .pytest_cache +mkdir -p "$OUTDIR" + +# --- Create isolated venv for build tools --- +if [[ ! -d "$VENV_DIR" ]]; then + echo "==> Creating build venv at $VENV_DIR" + $PYTHON -m venv "$VENV_DIR" +fi + +# shellcheck source=/dev/null +source "$VENV_DIR/bin/activate" +$PYTHON -m pip install --upgrade pip setuptools +$PYTHON -m pip install --upgrade build + +# If you use setuptools-scm, ensure it's present for version derivation +if grep -qiE 'setuptools[-_]scm' pyproject.toml; then + $PYTHON -m pip install --upgrade setuptools-scm +fi + +# --- Make builds more reproducible (optional but harmless) --- +export PYTHONHASHSEED=0 +export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(git log -1 --pretty=%ct 2>/dev/null || date +%s)}" + +# --- Build sdist and wheel --- +echo "==> Running python -m build" +$PYTHON -m build --sdist --wheel --outdir "$OUTDIR" $EXTRA_BUILD_ARGS + +# --- List outputs --- +echo "==> Produced artifacts:" +ls -lh "$OUTDIR" + +deactivate +echo "==> Build complete." diff --git a/tools/configuration.py b/tools/configuration.py index d780577e..21419320 100644 --- a/tools/configuration.py +++ b/tools/configuration.py @@ -7,6 +7,7 @@ TEMPLATES_DIR = PROJECT_DIR.joinpath("../templates") # Generated documents -GENERATED_DIR = PROJECT_DIR.joinpath("../generated") +GENERATED_DIR = PROJECT_DIR.joinpath("../site") +GENERATED_DOCS_DIR = GENERATED_DIR.joinpath("docs") XSD_FILE_PATH = PROJECT_DIR.joinpath("../xsd/xsd/NeTEx_publication.xsd") diff --git a/tools/expand_docs/expand_docs.py b/tools/expand_docs/expand_docs.py index 34a3e419..03333902 100644 --- a/tools/expand_docs/expand_docs.py +++ b/tools/expand_docs/expand_docs.py @@ -6,15 +6,17 @@ import shutil import argparse import re +from tools.configuration import DOCS_DIR, GENERATED_DOCS_DIR def copy_media_folder(input_folder, output_folder): """Copy media folder from input to output.""" media_src = os.path.join(input_folder, 'media') media_dst = os.path.join(output_folder, 'media') - + if os.path.exists(media_src): shutil.copytree(media_src, media_dst, dirs_exist_ok=True) + def include_xml_snippet(match, base_folder): """Include XML snippet content directly.""" snippet_path = match.group(1) @@ -23,12 +25,13 @@ def include_xml_snippet(match, base_folder): # base_folder is already the docs folder, so go up one level to project root project_root = os.path.abspath(os.path.join(base_folder, '..')) full_path = os.path.join(project_root, 'generated', 'xml-snippets', snippet_path) - + if os.path.exists(full_path): with open(full_path, 'r', encoding='utf-8') as f: return f"```xml\n{f.read()}\n```" return match.group(0) + def include_markdown_table(match, base_folder): """Include markdown table content directly.""" table_path = match.group(1) @@ -37,7 +40,7 @@ def include_markdown_table(match, base_folder): # base_folder is already the docs folder, so go up one level to project root project_root = os.path.abspath(os.path.join(base_folder, '..')) full_path = os.path.join(project_root, 'generated', 'markdown-examples', table_path) - + if os.path.exists(full_path): with open(full_path, 'r', encoding='utf-8') as f: content = f.read() @@ -54,43 +57,46 @@ def include_markdown_table(match, base_folder): return '\n'.join(table_lines) return match.group(0) + def process_markdown_file(input_path, output_path, base_folder): """Process a single markdown file.""" with open(input_path, 'r', encoding='utf-8') as f: content = f.read() - + # Process XML snippets - match the entire line prefix and link (flexible text) xml_pattern = r'(?:- )?\[.*?\]\(\.\./generated/xml-snippets/([^)]+\.xml)\)' content = re.sub(xml_pattern, lambda m: '\n\n' + include_xml_snippet(m, base_folder) + '\n\n', content) - + # Process markdown tables - match the entire line prefix and link (flexible text) md_pattern = r'(?:- )?\[.*?\]\(\.\./generated/markdown-examples/([^)]+\.md)\)' content = re.sub(md_pattern, lambda m: '\n\n' + include_markdown_table(m, base_folder) + '\n\n', content) - + # Write processed content with open(output_path, 'w', encoding='utf-8') as f: f.write(content) -def main(): - parser = argparse.ArgumentParser(description='Expand documentation by including examples and tables.') - parser.add_argument('--docs', required=True, help='Input documentation folder') - parser.add_argument('--out', required=True, help='Output folder') - args = parser.parse_args() - +def expand_docs(input_dir: str, output_dir: str): # Create output folder if it doesn't exist - os.makedirs(args.out, exist_ok=True) - + os.makedirs(output_dir, exist_ok=True) + # Copy media folder - copy_media_folder(args.docs, args.out) - + copy_media_folder(input_dir, output_dir) + # Process each markdown file - for filename in os.listdir(args.docs): + for filename in os.listdir(input_dir): if filename.endswith('.md'): - input_path = os.path.join(args.docs, filename) - output_path = os.path.join(args.out, filename) - process_markdown_file(input_path, output_path, args.docs) - - print(f"Documentation expanded successfully to {args.out}") + input_md = os.path.join(input_dir, filename) + output_md = os.path.join(output_dir, filename) + process_markdown_file(input_md, output_md, input_dir) + + print(f"Documentation expanded successfully to {output_dir}") + +def main(): + parser = argparse.ArgumentParser(description='Expand documentation by including examples and tables.') + parser.add_argument('--docs', default=DOCS_DIR, help=f"Input documentation folder (default = {DOCS_DIR})") + parser.add_argument('--out', default=GENERATED_DOCS_DIR, help=f"Output folder (default = {GENERATED_DOCS_DIR})") + args = parser.parse_args() + expand_docs(args.docs, args.out) if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/.github/workflows/main.yml b/tools/md2html/__init__.py similarity index 100% rename from .github/workflows/main.yml rename to tools/md2html/__init__.py diff --git a/tools/md2html/md2html.py b/tools/md2html/md2html.py new file mode 100644 index 00000000..5c5c78da --- /dev/null +++ b/tools/md2html/md2html.py @@ -0,0 +1,74 @@ +import argparse +import os +import re + +import markdown +from pygments.formatters import HtmlFormatter + +from tools.configuration import GENERATED_DOCS_DIR + +MD_LINK_PATTERN = re.compile(r'(\[.*\])(\(.*\.)(md)(.*\))') +MD_LINK_REPLACEMENT = r'\1\2html\4' + +def generate_html(md: str) -> str: + + md_with_html_links = MD_LINK_PATTERN.sub(MD_LINK_REPLACEMENT, md) + body = generate_html_body(md_with_html_links) + + # Generate CSS for code highlighting + code_css = HtmlFormatter().get_style_defs(".codehilite") + + # Wrap in a simple HTML template + return f""" + +
+ +