From 41e3164b3ebd6c2641492673af608d7a93130341 Mon Sep 17 00:00:00 2001 From: jnnr <32454596+jnnr@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:16:40 +0200 Subject: [PATCH 1/7] Run copier recopy --- .all-contributorsrc | 16 +++++++ .copier-answers.yml | 17 ++++--- .github/ISSUE_TEMPLATE/bug_report.md | 12 ++--- .github/ISSUE_TEMPLATE/config.yaml | 5 -- .github/ISSUE_TEMPLATE/config.yml | 11 +++++ .github/workflows/check-version.yml | 70 ++++++++++++++++++++++++++-- .github/workflows/pr-ci.yml | 8 ++-- .gitignore | 1 + .pre-commit-config.yaml | 6 +-- AUTHORS | 4 ++ CITATION.cff | 5 +- INTERFACE.yaml | 22 +++++---- README.md | 63 ++++++++++++++++++++----- config/README.md | 57 ++++------------------ pixi.toml | 34 +++++++++++++- tests/conftest.py | 12 +++++ tests/integration_test.py | 43 +++++++++++++++-- workflow/internal/config.schema.yaml | 8 ++++ 18 files changed, 287 insertions(+), 107 deletions(-) create mode 100644 .all-contributorsrc delete mode 100644 .github/ISSUE_TEMPLATE/config.yaml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 tests/conftest.py create mode 100644 workflow/internal/config.schema.yaml diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..03ee976 --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,16 @@ +{ + "projectName": "grid_builder", + "projectOwner": "PyPSA", + "repoType": "github", + "repoHost": "https://github.com", + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": false, + "commitConvention": "none", + "contributors": [], + "contributorsPerLine": 7, + "linkToUsage": false, + "contributorsSortAlphabetically": true +} diff --git a/.copier-answers.yml b/.copier-answers.yml index 5bb27d0..e52c88a 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,13 +1,18 @@ # Changes here will be overwritten by Copier # !!!!! DO NOT MANUALLY MODIFY THIS FILE !!!!! -_commit: latest +_commit: v1.0.3 _src_path: https://github.com/modelblocks-org/data-module-template.git -author_email: xiong@tu-berlin.de -author_family_name: Xiong -author_given_name: Bobby +authors: +- email: xiong@tu-berlin.de + family_name: Xiong + given_name: Bobby github_org: PyPSA license: MIT +maintainers: +- email: xiong@tu-berlin.de + family_name: Xiong + given_name: Bobby +module_code_name: grid_builder module_description: A modular Snakemake workflow for constructing and validating power grid models using OpenStreetMap data. -module_long_name: Global power grid builder -module_short_name: grid_builder +module_human_name: Global power grid builder diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 1b66ddc..6fe23c7 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,13 +8,13 @@ assignees: '' --- **Describe the bug** -A clear and concise description of what the bug is. +A clear and concise description of the bug. -**To Reproduce** +**Steps to reproduce** Steps to reproduce the behavior: 1. Go to '...' -2. Click on '....' -3. Scroll down to '....' +2. Run '...' +3. Provide '...' 4. See error **Expected behavior** @@ -23,9 +23,9 @@ A clear and concise description of what you expected to happen. **Screenshots** If applicable, add screenshots to help explain your problem. -**Desktop (please complete the following information):** +**Environment (please complete the following information):** - OS: [e.g. Linux Fedora 43, Windows 11...] - - Version [e.g. v0.1.1] + - Version: [e.g. v0.1.1] **Additional context** Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yaml b/.github/ISSUE_TEMPLATE/config.yaml deleted file mode 100644 index 28da94d..0000000 --- a/.github/ISSUE_TEMPLATE/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: General information - url: https://www.modelblocks.org/ - about: Please consult our website for general information on the Modelblocks methodology. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..b68f870 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Modelblocks project website + url: https://www.modelblocks.org/ + about: Consult our website for general information on our initiative. + - name: Documentation + url: https://modelblocks.readthedocs.io/ + about: Read our documentation for general questions and guidelines. + - name: Community chat + url: https://calliope-modelblocks.zulipchat.com + about: Reach out to our community on our official Zulip chat. diff --git a/.github/workflows/check-version.yml b/.github/workflows/check-version.yml index 7bde778..04e53c3 100644 --- a/.github/workflows/check-version.yml +++ b/.github/workflows/check-version.yml @@ -1,13 +1,75 @@ -# Check for changes in the upstream template. If changes are found, an issue is created -name: Template check. +# Check for changes in the upstream template. If changes are found, an issue is created. +name: Template check + on: workflow_dispatch: schedule: - cron: '0 0 1 * *' # Runs at 00:00 UTC on the 1st day of every month +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + jobs: - copier-update: + copier-check-update: + runs-on: ubuntu-latest permissions: contents: read issues: write - uses: modelblocks-org/data-module-template/.github/workflows/template-check-version.yml@latest + env: + ISSUE_TITLE: Template update available + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Check for template updates + id: copier_check_update + run: | + update_json="$(uvx --from 'copier>=9.15.2' copier check-update --output-format json)" + echo "$update_json" + { + echo "update_available=$(jq -r '.update_available // false' <<< "$update_json")" + echo "current_version=$(jq -r '.current_version // "unknown"' <<< "$update_json")" + echo "latest_version=$(jq -r '.latest_version // "unknown"' <<< "$update_json")" + } >> "$GITHUB_OUTPUT" + + - name: Check for existing update issue + if: steps.copier_check_update.outputs.update_available == 'true' + id: existing_issue + run: | + issue_count="$(gh issue list \ + --repo "${{ github.repository }}" \ + --state open \ + --search "$ISSUE_TITLE in:title" \ + --json title \ + --jq "map(select(.title == \"$ISSUE_TITLE\")) | length")" + if [[ "$issue_count" -gt 0 ]]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Open update issue + if: steps.copier_check_update.outputs.update_available == 'true' && steps.existing_issue.outputs.exists != 'true' + run: | + gh issue create \ + --repo "${{ github.repository }}" \ + --title "$ISSUE_TITLE" \ + --body "A new version of the Modelblocks data module template is available. + + Current template version: ${{ steps.copier_check_update.outputs.current_version }} + Latest template version: ${{ steps.copier_check_update.outputs.latest_version }} + + Please update this project using: + + \`\`\`shell + copier update --skip-answered --defaults + \`\`\` + + Review the resulting changes before merging them." + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index e3e0aaa..48e818b 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -14,9 +14,9 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.3 + uses: prefix-dev/setup-pixi@v0.9.6 - name: Run integration tests id: tests run: pixi run --locked test @@ -28,10 +28,10 @@ jobs: continue-on-error: true - name: Save integration logs if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: integration-test-logs-${{ matrix.os }} - path: tests/integration/logs + path: tests/integration/resources/module/logs if-no-files-found: ignore retention-days: 30 - name: Fail if integration or linting failed diff --git a/.gitignore b/.gitignore index cd24cc6..95f97f6 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ __pycache__ ### Environments .pixi/ +!.pixi/.gitignore ### Snakemake .snakemake/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 238e988..0156cb4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,15 +25,15 @@ repos: # Python file formatting - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.12 + rev: v0.15.18 hooks: - - id: ruff + - id: ruff-check args: [--fix, --exit-non-zero-on-fix] - id: ruff-format # Snakemake file formatting - repo: https://github.com/snakemake/snakefmt - rev: v1.1.0 + rev: v2.0.2 hooks: - id: snakefmt diff --git a/AUTHORS b/AUTHORS index e856cef..19fff9c 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,4 +2,8 @@ This is the list of contributors for copyright purposes. This does not necessarily list everyone who has contributed to this software's code or documentation. For a full contributor list, see: + + +Bobby Xiong, + diff --git a/CITATION.cff b/CITATION.cff index be879f0..ed890de 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,11 +2,12 @@ # For more information, see: # https://citation-file-format.github.io/ cff-version: 1.2.0 -message: "If you use this software or data produced by it, please cite it using the metadata from this file." +message: If you use this software or data produced by it, please cite it using the metadata from this file. title: "grid-builder: Global power grid builder" repository: "https://github.com/PyPSA/grid-builder" -license: "MIT" +license: MIT authors: - given-names: "Bobby" family-names: "Xiong" + email: "xiong@tu-berlin.de" orcid: "https://orcid.org/0000-0003-2854-0730" diff --git a/INTERFACE.yaml b/INTERFACE.yaml index 03794ca..11b2bdb 100644 --- a/INTERFACE.yaml +++ b/INTERFACE.yaml @@ -1,4 +1,6 @@ # Module Input-Output structure for automated docs generation +convention_version: v1.0.0 + pathvars: snakemake_defaults: logs: @@ -6,14 +8,16 @@ pathvars: description: location of snakemake log files. resources: default: "" - description: >- - Raw OSM files are written to retrieve/{country}_{feature}.json. - Clean features are written below clean; generic network outputs - below build (buses/lines/transformers as CSV under build/csv - and GeoJSON, including substation polygons, under build/geojson). - An interactive PyDeck map of the network, and the default target - of this workflow, is written to map.html. + description: "location of module resource (input) files." results: default: "" - description: location of module result files. - user_resources: {} + description: "location of module result (output) files." + retrieve: + default: + description: >- + Raw OSM files are written to retrieve/{country}_{feature}.json. + Clean features are written below clean; generic network outputs + below build (buses/lines/transformers as CSV under build/csv + and GeoJSON, including substation polygons, under build/geojson). + An interactive PyDeck map of the network, and the default target + of this workflow, is written to map.html. \ No newline at end of file diff --git a/README.md b/README.md index 821a651..884cbf6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # grid-builder -A modular Snakemake workflow for retrieving OpenStreetMap power infrastructure. +A modular Snakemake workflow for constructing and validating power grid models using OpenStreetMap data. +

@@ -11,16 +12,20 @@ A modular Snakemake workflow for retrieving OpenStreetMap power infrastructure.

## About + -`grid-builder` is a modular `snakemake` workflow that retrieves OpenStreetMap power infrastructure and builds a generic high-voltage network. It can be imported into another `snakemake` workflow. +This is a modular `snakemake` workflow created as part of the [Modelblocks project](https://www.modelblocks.org/). It can be imported directly into any `snakemake` workflow. -The workflow retains AC substations, overhead lines, and cables at configured voltage levels, then creates generic buses, connected line segments, and voltage-pair transformers. The outputs preserve OSM provenance and geometry but contain no PyPSA-specific line types, capacities, or electrical-component assumptions. - -This module follows the Modelblocks conventions (https://www.modelblocks.org). For more information, consult the [integration example](./tests/integration/Snakefile) and the `snakemake` [modularisation documentation](https://snakemake.readthedocs.io/en/stable/snakefiles/modularization.html). +For more information, please consult the Modelblocks [documentation](https://modelblocks.readthedocs.io/en/latest/), +the [integration example](./tests/integration/Snakefile), +and the `snakemake` [documentation](https://snakemake.readthedocs.io/en/stable/snakefiles/modularization.html). ## Overview + + +`grid-builder` retrieves OpenStreetMap power infrastructure and builds a generic high-voltage network. The workflow retains AC substations, overhead lines, and cables at configured voltage levels, then creates generic buses, connected line segments, and voltage-pair transformers. The outputs preserve OSM provenance and geometry but contain no PyPSA-specific line types, capacities, or electrical-component assumptions. -Currently implemented: +Data processing steps: 1. Retrieve OSM substations, lines, cables, and (optionally) circuit relations by country, either from a cached local Geofabrik PBF extract or the live Overpass API. 2. Clean the raw retrieval output, filtering voltage, frequency, construction status, and future assets, and grouping relation member ways into one line per real-world circuit. @@ -28,10 +33,12 @@ Currently implemented: 4. Build a self-contained interactive map of the resulting network (`map.html`), with layer toggles, voltage/text filtering, and click-through OSM links — this is the workflow's default target. ## Configuration + -Configuration lives in [`config/config.yaml`](./config/config.yaml), validated against a generated JSON schema. See the configuration [README](./config/README.md) for the available controls, including retrieval backends, regional overrides, and personal/local settings. +Please consult the configuration [README](./config/README.md) and the [configuration example](./config/config.yaml) for a general overview on the configuration options of this module. ## Input / output structure + Please consult the [interface file](./INTERFACE.yaml) for more information. @@ -56,6 +63,7 @@ DC assets (links, converters, switching stations) are out of scope: this workflo builds a generic AC topology only, with no PyPSA-specific line types or capacities. ## Development + We use [`pixi`](https://pixi.sh/) as our package manager for development. Once installed, run the following to clone this repository and install all dependencies. @@ -63,9 +71,24 @@ Once installed, run the following to clone this repository and install all depen ```shell git clone git@github.com:PyPSA/grid-builder.git cd grid-builder -pixi install --locked +pixi install --all ``` +Please be aware that this is a multi-environment project (see [pixi.toml](./pixi.toml) for details). +- `default`: used for development and integration testing. +Because it contains `Snakemake`, `conda` and `pytest` as dependencies it **should not be used** in `Snakemake` rules. +- `module`: contains minimal dependencies used in `Snakemake` rules. +If modified, be sure to export it to `Snakemake` so it can be recreated by module users: + +```shell +# create module.yaml and conda-spec pin files in workflow/envs/ +pixi run export-snakemake-env module +``` + + +## Testing + + For testing, simply run: ```shell @@ -76,9 +99,9 @@ pixi run --locked test To test a minimal example of a workflow using this module: ```shell -pixi shell # activate this project's environment -cd tests/integration/ # navigate to the integration example -snakemake --use-conda --cores 2 # run the workflow! +pixi shell # activate this project's environment +cd tests/integration/ # navigate to the integration example +snakemake --use-conda --cores 2 # run the workflow! ``` The Pixi environment supplies Snakemake and the configuration-validation @@ -104,8 +127,24 @@ rebuild the installed environment with `pixi reinstall --locked`. `grid-builder` is released as free software under the [MIT](LICENSE) license. Different licenses and terms of use may apply to input data, e.g. OpenStreetMap data is subject to the [Open Database License](https://opendatacommons.org/licenses/odbl). -## References & related work +## References + + +This module is based on the following research and datasets: * Jonas Hörsch et al. 2018. PyPSA-Eur: An open optimisation model of the European transmission system, *Energy Strategy Reviews*, Volume 22. https://doi.org/10.1016/j.esr.2018.08.012 * Maximilian Parzen et al. 2023. PyPSA-Earth: A new global open energy system optimization model demonstrated in Africa, *Applied Energy*, Volume 341. https://doi.org/10.1016/j.apenergy.2023.121096 * Bobby Xiong et al. 2025. Modelling the high-voltage grid using open data for Europe and beyond. *Sci Data* 12, 277. https://doi.org/10.1038/s41597-025-04550-7 + +## Contributors ✨ + +Thanks goes to these wonderful people, sorted alphabetically ([emoji key](https://allcontributors.org/en/reference/emoji-key/)): + + + + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/config/README.md b/config/README.md index e5166c9..720820c 100644 --- a/config/README.md +++ b/config/README.md @@ -1,49 +1,8 @@ -Set `countries` to the ISO country codes that define the retrieval scope. The -workflow first loads the default `config/config.yaml`, then an optional -`config/regions/config..yaml` file for every selected country. A `regions` -mapping in the calling configuration overrides values from those regional files. - -`retrieve.source` picks the retrieval backend: `geofabrik` reads a cached local -PBF extract (`retrieve_osm_pbf.py`), `overpass` queries the live Overpass API -(`retrieve_osm_overpass.py`). Both produce the same output shape, so -`clean` doesn't need to know which one ran. `network.include_relations` -decides whether the network should consider `route=power`/`power=circuit` -relations, grouping their member ways into one line per real-world circuit; -retrieval respects this too, so relations aren't fetched at all when it's off. -`network` also controls the minimum retained AC voltage, station merge buffer -radius, construction filtering, and planned-asset cutoff date. - -The [BE+NL example](./examples/config.BE-NL.yaml) is a small European development -scope. Country files under `config/regions` are intentionally small defaults for -now; community-maintained local corrections belong there rather than in workflow -code. - -`interactive_map` controls the size of `map.html`: `coordinate_decimals` rounds -embedded coordinates, and `simplify_geometries` sets per-geometry-type -Douglas-Peucker tolerances (in metres) for station polygons, bus polygons, and -lines, or disables simplification entirely via `simplify_geometries.enable`. - -### Personal settings and Overpass fair use - -Keep `config/config.yaml` as pure defaults — a test enforces that it matches the -schema, and it's tracked in git, so it's not the place for anything -environment- or person-specific. For local overrides (a custom Overpass -endpoint, contact details, a smaller `countries` scope for development), create -an untracked `config/config.local.yaml` and pass it alongside the default: - -```shell -snakemake --configfile config/config.local.yaml ... -``` - -Snakemake deep-merges it on top of `config/config.yaml`, so you only need to -list the keys you're overriding. - -If you use `retrieve.source: overpass`, set `retrieve.overpass_api.user_agent` -to your own project name, contact email, and website. The [Overpass API fair -use policy](https://wiki.openstreetmap.org/wiki/Overpass_API#Fair_use_policy) -expects automated queries to be identifiable and reachable; a generic or -missing user agent risks being rate-limited or blocked. `retrieve.overpass_api.url` -also lets you point at your own or a faster mirror instance instead of the -shared public endpoint, without touching the checked-in default. - -The generated [schema](./config.schema.json) describes every option. +We recommend consulting the following before using this module: +- `config/config.yaml`: a generic example configuration of this module. +- `workflow/internal/config.schema.yaml`: a schematic overview of all the configuration options of this module. +- `INTERFACE.yaml`: lists module input and output files, and their default locations. +- `tests/integration/Snakefile`: an example of how to call this module from another workflow. + +This data module is part of the [Modelblocks](https://www.modelblocks.org/) project. +Please consult the [Modelblocks documentation](https://modelblocks.readthedocs.io/) for more details. diff --git a/pixi.toml b/pixi.toml index f095350..f02abd2 100644 --- a/pixi.toml +++ b/pixi.toml @@ -1,7 +1,7 @@ [workspace] name = "grid_builder" authors = ["See AUTHORS file"] -description = "A modular Snakemake workflow for retrieving OpenStreetMap power infrastructure." +description = "A modular Snakemake workflow for constructing and validating power grid models using OpenStreetMap data." license = "MIT" readme = "README.md" channels = ["conda-forge", "bioconda"] @@ -9,12 +9,13 @@ platforms = ["win-64", "linux-64", "osx-arm64"] homepage = "https://www.modelblocks.org/" [dependencies] -clio-tools = ">=2026.03.30" +clio-tools = ">=2026.06.23" conda = ">=25.0.0" ipdb = ">=0.13.13" ipykernel = ">=6.29.5" pydantic = ">=2.0" "ruamel.yaml" = ">=0.18" +jsonschema = ">=4.0.0" mypy = ">=1.15.0" pytest = ">=8.3.5" python = ">=3.12,<3.13" @@ -24,6 +25,8 @@ ruff = ">=0.9.9" snakefmt = ">=0.10.2" snakemake-minimal = ">=9.19.0" pytz = ">=2026.1.post1" + +[feature.module.dependencies] earth-osm = ">=3.0.2" geopandas = ">=1.0,<2" networkx = ">=3,<4" @@ -37,6 +40,9 @@ requests = ">=2,<3" shapely = ">=2,<3" pre-commit = ">=4.6.2,<5" +[environments] +module = { features = ["module"], no-default-feature = true } + [tasks] test = {cmd = "pytest tests"} test-integration = {cmd = "pytest tests/integration_test.py"} @@ -44,3 +50,27 @@ generate-config = {cmd = "python workflow/scripts/_schema.py"} check-python = {cmd = "ruff check workflow/scripts tests && ruff format --check workflow/scripts tests && mypy workflow/scripts"} check-workflow = {cmd = "snakefmt --check workflow tests/integration/Snakefile && snakemake --lint --snakefile workflow/Snakefile && snakemake --lint --snakefile tests/integration/Snakefile"} lint = {depends-on = ["check-python", "check-workflow"]} + +[tasks.export-snakemake-env] +description = "Export one Pixi environment as Snakemake-compatible conda files" +args = ["env", { arg = "outdir", default = "workflow/envs" }] +depends-on = [ + { task = "_export-snakemake-pin", args = ["{{ env }}", "win-64", "{{ outdir }}"] }, + { task = "_export-snakemake-pin", args = ["{{ env }}", "linux-64", "{{ outdir }}"] }, + { task = "_export-snakemake-pin", args = ["{{ env }}", "osx-arm64", "{{ outdir }}"] }, +] +cmd = "pixi workspace export conda-environment --environment '{{ env }}' '{{ outdir }}/{{ env }}.yaml'" + +[tasks._export-snakemake-pin] +description = "Export one Pixi environment/platform as a Snakemake-compatible pin file" +args = ["env", "platform", "outdir"] +depends-on = [ + { task = "_export-conda-explicit-spec", args = ["{{ env }}", "{{ platform }}", "{{ outdir }}"] }, +] +cmd = "mv '{{ outdir }}/{{ env }}_{{ platform }}_conda_spec.txt' '{{ outdir }}/{{ env }}.{{ platform }}.pin.txt'" + +[tasks._export-conda-explicit-spec] +description = "Export one Pixi environment/platform as an explicit conda spec file" +args = ["env", "platform", "outdir"] +cmd = "pixi workspace export conda-explicit-spec --environment '{{ env }}' --platform '{{ platform }}' '{{ outdir }}'" + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f5a5699 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,12 @@ +"""Shared pytest fixtures.""" + +from pathlib import Path + +import pytest + + +@pytest.fixture(scope="module") +def module_path(): + """Parent directory of the project.""" + # If your module needs files in resources/user/, place automated downloads here. + return Path(__file__).parent.parent diff --git a/tests/integration_test.py b/tests/integration_test.py index 3c9af85..85641ad 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -11,6 +11,7 @@ import subprocess import sys import tempfile +import tomllib from pathlib import Path import pytest @@ -18,9 +19,34 @@ @pytest.fixture(scope="module") -def module_path(): - """Parent directory of the project.""" - return Path(__file__).parent.parent +def pixi_platforms(module_path) -> list[str]: + """Pixi platforms defined for this project.""" + with (module_path / "pixi.toml").open("rb") as pixi_config: + return tomllib.load(pixi_config)["workspace"]["platforms"] + + +def test_snakemake_environments(module_path, pixi_platforms, tmp_path): + """All Snakemake environment files should be based on pixi counterparts.""" + env_dir = module_path / "workflow/envs" + env_files = sorted(env_dir.glob("*.yaml")) + assert env_files, f"No conda environments found in {module_path}." + + for env_file in env_files: + env_name = env_file.stem + + output_dir = tmp_path / env_name + subprocess.run( + ["pixi", "run", "export-snakemake-env", env_name, str(output_dir)], + check=True, + cwd=module_path, + ) + + generated_yaml = output_dir / env_file.name + assert generated_yaml.read_text() == env_file.read_text() + + for platform in pixi_platforms: + pin_file = env_dir / f"{env_name}.{platform}.pin.txt" + assert pin_file.exists(), f"{env_name} has no conda pins for {platform}" def test_interface_file(module_path): @@ -37,8 +63,7 @@ def test_interface_file(module_path): "LICENSE", "README.md", "config/config.yaml", - "config/config.schema.json", - "config/examples/config.BE-NL.yaml", + "workflow/internal/config.schema.yaml", "tests/integration/Snakefile", ], ) @@ -47,6 +72,14 @@ def test_standard_file_existance(module_path, file): assert Path(module_path / file).exists() +def test_snakemake_all_failure(module_path): + """The snakemake 'all' rule should return an error by default.""" + process = subprocess.run( + "snakemake --cores 1", shell=True, cwd=module_path, capture_output=True + ) + assert "INVALID" in str(process.stderr) + + def test_snakemake_integration_testing(module_path): """Run retrieval from scratch in its Conda environment and validate outputs.""" log_dir = module_path / "tests/integration/logs" diff --git a/workflow/internal/config.schema.yaml b/workflow/internal/config.schema.yaml new file mode 100644 index 0000000..c0b8a46 --- /dev/null +++ b/workflow/internal/config.schema.yaml @@ -0,0 +1,8 @@ +$schema: "https://json-schema.org/draft/2020-12/schema" +description: "Schema for user-provided configuration files." +type: object +additionalProperties: false +properties: + dummy_text: + description: A template example. Should be deleted in real applications. + type: string From 294097a9c4098c4ecc21b2252716bab45a73fcd9 Mon Sep 17 00:00:00 2001 From: jnnr <32454596+jnnr@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:19:17 +0200 Subject: [PATCH 2/7] Add python modules to workflow.source_path --- workflow/Snakefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index b8ed000..9226957 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -4,6 +4,11 @@ from pathlib import Path import yaml from snakemake.utils import min_version +# Load python files in scripts that are used in other scripts, but not run itself. +# This is necessary for integrating the snakemake module via reference to GitHub. +workflow.source_path("scripts/_helpers.py") +workflow.source_path("scripts/_schema.py") + sys.path.insert(0, workflow.basedir) from scripts._schema import load_region_configs, validate_config From bd35a17f2cca4dd6f1c2e6756202833c0a350d15 Mon Sep 17 00:00:00 2001 From: jnnr <32454596+jnnr@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:44:15 +0200 Subject: [PATCH 3/7] Attempt to fix python import --- workflow/Snakefile | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 9226957..1489f93 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -9,7 +9,6 @@ from snakemake.utils import min_version workflow.source_path("scripts/_helpers.py") workflow.source_path("scripts/_schema.py") -sys.path.insert(0, workflow.basedir) from scripts._schema import load_region_configs, validate_config min_version("9.19") From 494cb9a12b79777534cae80cdb64ed0ac61b6f39 Mon Sep 17 00:00:00 2001 From: jnnr <32454596+jnnr@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:37:58 +0200 Subject: [PATCH 4/7] Clean up Snakefile --- workflow/Snakefile | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 1489f93..2d9909e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,37 +1,35 @@ -import sys -from pathlib import Path - import yaml -from snakemake.utils import min_version - -# Load python files in scripts that are used in other scripts, but not run itself. -# This is necessary for integrating the snakemake module via reference to GitHub. -workflow.source_path("scripts/_helpers.py") -workflow.source_path("scripts/_schema.py") -from scripts._schema import load_region_configs, validate_config +from snakemake.utils import min_version, validate min_version("9.19") -# Define pathvars to expose OSM retrieval outputs for downstream use. +# !!!!! IMPORTANT !!!!! +# Define pathvars for all user resource files and result files +# This allows users to re-wire how the module is used in their workflow with ease. pathvars: # OSM retrieval outputs by country and feature osm_retrieve="/retrieve", -# Default configuration file generated from pydantic schema. +# Load the example configuration. This will be overridden by users. configfile: workflow.source_path("../config/config.yaml") -config = validate_config( - load_region_configs(config, Path(workflow.basedir).parent / "config" / "regions") -) +# Validate the configuration using the schema file. +validate(config, workflow.source_path("internal/config.schema.yaml")) + # Load internal settings separately so users cannot modify them. with open(workflow.source_path("internal/settings.yaml"), "r") as f: internal = yaml.safe_load(f) +# Load python files in scripts that are used in other scripts, but not run itself. +# This is necessary for integrating the snakemake module via reference to GitHub. +workflow.source_path("scripts/_helpers.py") +workflow.source_path("scripts/_schema.py") + include: "rules/retrieve.smk" include: "rules/network.smk" From 5882f3dd94a127defd50f2fa2476e278d90b3563 Mon Sep 17 00:00:00 2001 From: Jann Launer <32454596+jnnr@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:07:47 +0200 Subject: [PATCH 5/7] Use template's config schema --- workflow/internal/config.schema.yaml | 212 ++++++++++++++++++++++++++- workflow/rules/network.smk | 8 +- workflow/rules/plot.smk | 4 +- workflow/rules/retrieve.smk | 2 +- 4 files changed, 211 insertions(+), 15 deletions(-) diff --git a/workflow/internal/config.schema.yaml b/workflow/internal/config.schema.yaml index c0b8a46..8ede2a1 100644 --- a/workflow/internal/config.schema.yaml +++ b/workflow/internal/config.schema.yaml @@ -1,8 +1,204 @@ -$schema: "https://json-schema.org/draft/2020-12/schema" -description: "Schema for user-provided configuration files." -type: object -additionalProperties: false -properties: - dummy_text: - description: A template example. Should be deleted in real applications. - type: string + $schema: "https://json-schema.org/draft/2020-12/schema" + title: ConfigSchema + description: Top-level grid-builder config. + type: object + additionalProperties: false + properties: + countries: + description: List of countries to retrieve OSM data for + type: array + minItems: 1 + items: + type: string + default: + - BE + + retrieve: + description: Settings for retrieve_osm_pbf.py/retrieve_osm_overpass.py. + type: object + additionalProperties: false + properties: + source: + description: Retrieval backend for OSM data + type: string + enum: + - geofabrik + - overpass + default: geofabrik + force_redownload: + description: Force refresh of cached data + type: boolean + default: false + target_date: + description: Optional historical date for data retrieval in ISO 8601 datetime format. Only honoured for retrieve.source geofabrik. + anyOf: + - type: string + format: date-time + - type: "null" + default: null + overpass_api: + description: Settings for retrieve_osm_pbf.py/retrieve_osm_overpass.py's own Overpass API client. + type: object + additionalProperties: false + properties: + url: + description: Overpass API endpoint to query + type: string + default: "https://overpass-api.de/api/interpreter" + max_tries: + description: Maximum number of attempts per query before giving up + type: integer + minimum: 1 + default: 5 + timeout: + description: Per-request timeout in seconds + type: integer + exclusiveMinimum: 0 + default: 600 + user_agent: + description: Identifies this tool to the Overpass API, per its fair-use policy. + type: object + additionalProperties: false + properties: + project_name: + description: Project name sent in the User-Agent header + type: string + default: grid-builder + email: + description: Contact email sent in the User-Agent header + type: string + default: "" + website: + description: Project URL sent in the User-Agent header + type: string + default: "https://github.com/pypsa/grid-builder" + + network: + description: Global assumptions for cleaning and connecting OSM grid features. + type: object + additionalProperties: false + properties: + include_relations: + description: Whether the network should consider OSM route=power/power=circuit relations, grouping their member ways into a single line matching the relation's real-world circuit; retrieval respects this too, so relations are not fetched when disabled. + type: boolean + default: true + minimum_voltage_kv: + description: Minimum nominal AC voltage retained from OSM, in kV + type: number + exclusiveMinimum: 0 + default: 220.0 + frequency_hz: + description: AC/DC frequency in Hz, used to classify and normalise OSM frequency tags. + type: object + additionalProperties: false + properties: + AC: + description: AC frequency in Hz (50 for most of the world; 60 for the Americas and parts of Asia) + type: number + exclusiveMinimum: 0 + default: 50.0 + DC: + description: Frequency tag value OSM uses to mark a DC line/converter + type: number + minimum: 0 + default: 0.0 + station_merge_radius_m: + description: Buffer radius used to merge nearby substations and line endpoints, in metres + type: number + exclusiveMinimum: 0 + default: 500.0 + remove_under_construction: + description: Whether assets tagged as under construction are dropped + type: boolean + default: true + remove_after: + description: Exclude assets with a later planned start date; null disables this filter + anyOf: + - type: string + format: date + - type: "null" + default: "2026-12-31" + + regions: + description: Country-specific network overrides loaded from config/regions + type: object + additionalProperties: + description: Optional country-specific overrides loaded from config/regions. + type: object + additionalProperties: false + properties: + minimum_voltage_kv: + anyOf: + - type: number + exclusiveMinimum: 0 + - type: "null" + default: null + frequency_hz: + anyOf: + - description: Optional per-country AC/DC override; unset fields fall back to network.frequency_hz. + type: object + additionalProperties: false + properties: + AC: + anyOf: + - type: number + exclusiveMinimum: 0 + - type: "null" + default: null + DC: + anyOf: + - type: number + minimum: 0 + - type: "null" + default: null + - type: "null" + default: null + + interactive_map: + description: Geometry simplification and coordinate rounding for build_interactive_map.py. + type: object + additionalProperties: false + properties: + coordinate_decimals: + description: Decimal places kept for coordinates embedded in the interactive map; 5 is about 1.1 m of precision at the equator, comfortably below every simplification tolerance below. + type: integer + minimum: 0 + default: 5 + simplify_geometries: + description: Douglas-Peucker simplification tolerances for the interactive map. + type: object + additionalProperties: false + properties: + enable: + description: Whether to simplify station/bus-polygon/line geometries before embedding them in the interactive map + type: boolean + default: true + stations_m: + description: Simplification tolerance for station polygon outlines, in metres + type: number + minimum: 0 + default: 100.0 + buses_polygon_m: + description: Simplification tolerance for individual bus/substation footprint polygons, in metres + type: number + minimum: 0 + default: 5.0 + lines_m: + description: Simplification tolerance for line geometries, in metres + type: number + minimum: 0 + default: 30.0 + + crs: + description: Coordinate reference systems used throughout clean/build_network. + type: object + additionalProperties: false + properties: + geo: + description: Geographic CRS used to store and exchange coordinates + type: string + default: "EPSG:4326" + distance: + description: Equal-area/equal-distance CRS used for buffering and length calculations; must suit the geographic extent in use (the default, ETRS89-LAEA, covers Europe) + type: string + default: "EPSG:3035" diff --git a/workflow/rules/network.smk b/workflow/rules/network.smk index 3d5476b..1619625 100644 --- a/workflow/rules/network.smk +++ b/workflow/rules/network.smk @@ -43,12 +43,12 @@ rule clean: "../envs/network.yaml" threads: 1 params: - network=config["network"].model_dump(mode="json"), + network=config["network"], regions={ - code: value.model_dump(mode="json") + code: value for code, value in config["regions"].items() }, - crs=config["crs"].model_dump(mode="json"), + crs=config["crs"], message: "Cleaning retrieved OSM power features." script: @@ -78,7 +78,7 @@ rule build_network: station_merge_radius_m=config["network"]["station_merge_radius_m"], remove_under_construction=config["network"]["remove_under_construction"], remove_after=config["network"]["remove_after"], - crs=config["crs"].model_dump(mode="json"), + crs=config["crs"], message: "Building a connected generic OSM network." script: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 3c06d71..163fbd5 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -18,8 +18,8 @@ rule build_interactive_map: "../envs/network.yaml" threads: 1 params: - crs=config["crs"].model_dump(mode="json"), - interactive_map=config["interactive_map"].model_dump(mode="json"), + crs=config["crs"], + interactive_map=config["interactive_map"], message: "Building an interactive OSM network map." script: diff --git a/workflow/rules/retrieve.smk b/workflow/rules/retrieve.smk index cc6144f..26eab40 100644 --- a/workflow/rules/retrieve.smk +++ b/workflow/rules/retrieve.smk @@ -58,7 +58,7 @@ elif config["retrieve"]["source"] == "overpass": threads: 1 params: include_relations=config["network"]["include_relations"], - overpass_api=config["retrieve"]["overpass_api"].model_dump(mode="json"), + overpass_api=config["retrieve"]["overpass_api"], message: "Retrieve OSM power features for one country from the Overpass API." script: From 03deec2804d24defef69eb319bc60452b1ac62d6 Mon Sep 17 00:00:00 2001 From: Jann Launer <32454596+jnnr@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:36:37 +0200 Subject: [PATCH 6/7] Adapt python imports --- workflow/scripts/_helpers.py | 2 +- workflow/scripts/build_interactive_map.py | 4 ++-- workflow/scripts/build_network.py | 4 ++-- workflow/scripts/clean.py | 4 ++-- workflow/scripts/retrieve_osm_overpass.py | 4 ++-- workflow/scripts/retrieve_osm_pbf.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/workflow/scripts/_helpers.py b/workflow/scripts/_helpers.py index 0ca446b..935306d 100644 --- a/workflow/scripts/_helpers.py +++ b/workflow/scripts/_helpers.py @@ -7,7 +7,7 @@ Logging setup, internal data loading, shared geometry constants, and mock_snakemake for testing. -Other scripts import this module as ``from scripts._helpers import ...``, +Other scripts import this module as ``from _helpers import ...``, not ``workflow.scripts._helpers``. That plain form resolves in both places these scripts run: under a real Snakemake ``script:`` execution, Snakemake puts ``workflow/`` on ``sys.path`` diff --git a/workflow/scripts/build_interactive_map.py b/workflow/scripts/build_interactive_map.py index 4fb5e97..b2e1859 100644 --- a/workflow/scripts/build_interactive_map.py +++ b/workflow/scripts/build_interactive_map.py @@ -23,7 +23,7 @@ import geopandas as gpd import pandas as pd import pydeck as pdk -from scripts._helpers import configure_logging, load_internal_yaml +from _helpers import configure_logging, load_internal_yaml if TYPE_CHECKING: snakemake: Any @@ -1080,7 +1080,7 @@ def compress_json_in_script(match: re.Match[str]) -> str: if __name__ == "__main__": if "snakemake" not in globals(): - from scripts._helpers import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_interactive_map") configure_logging(snakemake.log[0]) diff --git a/workflow/scripts/build_network.py b/workflow/scripts/build_network.py index e4803af..31334e7 100644 --- a/workflow/scripts/build_network.py +++ b/workflow/scripts/build_network.py @@ -23,7 +23,7 @@ import numpy as np import pandas as pd from pyproj import Transformer -from scripts._helpers import BUS_TOL, configure_logging +from _helpers import BUS_TOL, configure_logging from shapely import get_point from shapely.algorithms.polylabel import polylabel from shapely.geometry import LineString, MultiLineString, Point @@ -940,7 +940,7 @@ def _write_components( if __name__ == "__main__": if "snakemake" not in globals(): - from scripts._helpers import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("build_network") diff --git a/workflow/scripts/clean.py b/workflow/scripts/clean.py index 32f5fcd..6ac11f7 100644 --- a/workflow/scripts/clean.py +++ b/workflow/scripts/clean.py @@ -27,7 +27,7 @@ import geopandas as gpd import numpy as np import pandas as pd -from scripts._helpers import BUS_TOL, configure_logging, load_internal_yaml +from _helpers import BUS_TOL, configure_logging, load_internal_yaml from shapely.algorithms.polylabel import polylabel from shapely.geometry import LineString, MultiLineString, Point, Polygon from shapely.ops import linemerge, unary_union @@ -1197,7 +1197,7 @@ def clean( if __name__ == "__main__": if "snakemake" not in globals(): - from scripts._helpers import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("clean") diff --git a/workflow/scripts/retrieve_osm_overpass.py b/workflow/scripts/retrieve_osm_overpass.py index 6495dde..82a11fb 100644 --- a/workflow/scripts/retrieve_osm_overpass.py +++ b/workflow/scripts/retrieve_osm_overpass.py @@ -29,7 +29,7 @@ import requests from earth_osm.regions import get_region_tuple from requests.adapters import HTTPAdapter -from scripts._helpers import configure_logging +from _helpers import configure_logging from urllib3.util.retry import Retry if TYPE_CHECKING: @@ -172,7 +172,7 @@ def retrieve_from_overpass( if __name__ == "__main__": if "snakemake" not in globals(): - from scripts._helpers import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("retrieve_osm_overpass", country="BE") diff --git a/workflow/scripts/retrieve_osm_pbf.py b/workflow/scripts/retrieve_osm_pbf.py index ee1cadb..c0302c2 100644 --- a/workflow/scripts/retrieve_osm_pbf.py +++ b/workflow/scripts/retrieve_osm_pbf.py @@ -42,7 +42,7 @@ import osmium from earth_osm.regions import download_region_pbf, get_region_tuple -from scripts._helpers import configure_logging +from _helpers import configure_logging if TYPE_CHECKING: snakemake: Any @@ -334,7 +334,7 @@ def _extract_from_filtered_pbf( if __name__ == "__main__": if "snakemake" not in globals(): - from scripts._helpers import mock_snakemake + from _helpers import mock_snakemake snakemake = mock_snakemake("retrieve_osm_pbf", country="BE") From f0b2a13512516b60f74543d7fd31c91f946e5ac0 Mon Sep 17 00:00:00 2001 From: Jann Launer <32454596+jnnr@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:48:03 +0200 Subject: [PATCH 7/7] Add colors and tag_corrections to source_path --- workflow/Snakefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2d9909e..1f902e4 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -29,6 +29,8 @@ with open(workflow.source_path("internal/settings.yaml"), "r") as f: # This is necessary for integrating the snakemake module via reference to GitHub. workflow.source_path("scripts/_helpers.py") workflow.source_path("scripts/_schema.py") +workflow.source_path("internal/colors.yaml") +workflow.source_path("internal/tag_corrections.yaml") include: "rules/retrieve.smk"