From e6021f76855645472f54a25fe31f96e9ea545ea1 Mon Sep 17 00:00:00 2001 From: Damian Reeves <957246+DamianReeves@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:55:46 -0500 Subject: [PATCH] feat(release): split library and CLI publishing Publish morphir 0.0.1 as the dependency-free library and SDK. Release morphir-python 0.0.2 as the CLI, depending on the library rather than bundling its files. Use independent package tags and OIDC publishing jobs. Verify namespace ownership, isolated installs, migration from the old bundle, and the exact published core before CLI promotion. --- .config/mise/tasks/build | 7 +- .config/mise/tasks/clean | 1 + .config/mise/tasks/release-check | 169 +++++++++-------- .config/mise/tasks/typecheck | 2 +- .github/workflows/ci.yml | 11 +- .github/workflows/release.yml | 41 ++++- AGENTS.md | 12 +- README.md | 152 ++++++++++------ packages/morphir-tools/README.md | 9 +- packages/morphir-tools/pyproject.toml | 45 +++++ .../src/morphir_tools/__init__.py | 4 +- packages/morphir/README.md | 9 +- pyproject.toml | 47 +---- scripts/__init__.py | 1 + scripts/check_distributions.py | 171 ++++++++++++++++++ tests/tools/test_tools.py | 10 +- tests/unit/test_morphir.py | 8 + tests/unit/test_release_process.py | 104 +++++++++++ uv.lock | 22 ++- 19 files changed, 609 insertions(+), 216 deletions(-) create mode 100644 packages/morphir-tools/pyproject.toml create mode 100644 scripts/__init__.py create mode 100644 scripts/check_distributions.py create mode 100644 tests/unit/test_release_process.py diff --git a/.config/mise/tasks/build b/.config/mise/tasks/build index 059c4ada..b6716370 100755 --- a/.config/mise/tasks/build +++ b/.config/mise/tasks/build @@ -1,8 +1,11 @@ #!/usr/bin/env bash -#MISE description="Build the bundled morphir-python distribution" +#MISE description="Build the independent library and CLI distributions" set -euo pipefail +echo "Building morphir library and SDK..." +uv build packages/morphir --out-dir packages/morphir/dist + echo "Building morphir-python package..." -uv build --out-dir dist +uv build packages/morphir-tools --out-dir packages/morphir-tools/dist echo "Build complete!" diff --git a/.config/mise/tasks/clean b/.config/mise/tasks/clean index a15b0d5d..c818b58b 100755 --- a/.config/mise/tasks/clean +++ b/.config/mise/tasks/clean @@ -4,6 +4,7 @@ set -euo pipefail echo "Cleaning build artifacts..." rm -rf build/ dist/ .eggs/ +rm -rf packages/morphir/dist/ packages/morphir-tools/dist/ find . -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true find . -type f -name "*.py[cod]" -delete 2>/dev/null || true diff --git a/.config/mise/tasks/release-check b/.config/mise/tasks/release-check index 3b706c50..455eb42a 100755 --- a/.config/mise/tasks/release-check +++ b/.config/mise/tasks/release-check @@ -1,105 +1,102 @@ #!/usr/bin/env bash -#MISE description="Check built distributions and scan runtime dependencies" +#MISE description="Qualify separate library and CLI distributions for release" set -euo pipefail -# Keep the candidate environment separate from development and scanner dependencies. release_dir="$(pwd)/.dev/release-check" mkdir -p "$release_dir" - uv lock --check -python - <<'PY' -import os -import tarfile -import tomllib -import zipfile -from email.parser import BytesParser -from pathlib import Path - -for folder, name in ((".", "morphir-python"),): - directory = Path(folder) - project = tomllib.loads((directory / "pyproject.toml").read_text())["project"] - tag = os.environ.get("RELEASE_TAG", "") - if tag and tag not in (f"v{project['version']}", f"python-v{project['version']}"): - raise SystemExit(f"{tag} does not match {name} {project['version']}") - stem = name.replace("-", "_") + "-" + project["version"] - expected = {f"{stem}-py3-none-any.whl", f"{stem}.tar.gz"} - files = {p.name for p in (directory / "dist").iterdir() - if p.is_file() and p.name != ".gitignore"} - if files != expected: - raise SystemExit(f"Unexpected distributions for {name}: {files}") - for filename in expected: - path = directory / "dist" / filename - if path.suffix == ".whl": - with zipfile.ZipFile(path) as archive: - metadata = [n for n in archive.namelist() if n.endswith(".dist-info/METADATA")] - raw = archive.read(metadata[0]) - has_license = any(n.endswith("/licenses/LICENSE") for n in archive.namelist()) - for source in (Path("packages/morphir/src"), Path("packages/morphir-tools/src")): - expected_sources = {str(p.relative_to(source)) for p in source.rglob("*") - if p.is_file() and p.suffix in (".py", ".typed")} - missing = expected_sources - set(archive.namelist()) - if missing: - raise SystemExit(f"Missing bundled sources in {path}: {missing}") - else: - with tarfile.open(path) as archive: - member = archive.extractfile(f"{stem}/PKG-INFO") - if member is None: - raise SystemExit(f"Missing metadata in {path}") - raw = member.read() - has_license = f"{stem}/LICENSE" in archive.getnames() - metadata = BytesParser().parsebytes(raw) - if metadata["Name"] != name or metadata["Version"] != project["version"]: - raise SystemExit(f"Incorrect name or version in {path}") - if metadata["License-Expression"] != "Apache-2.0" or not has_license: - raise SystemExit(f"Missing Apache-2.0 license in {path}") - if any(req.startswith("morphir") for req in metadata.get_all("Requires-Dist", [])): - raise SystemExit(f"{path} must not depend on a separate Morphir distribution") -PY - +python scripts/check_distributions.py uvx --from twine==7.0.0 twine check --strict \ - dist/* - + packages/morphir/dist/* packages/morphir-tools/dist/* uv export --frozen --all-packages --no-default-groups --no-emit-workspace \ --no-hashes --output-file "$release_dir/constraints.txt" -uv venv --clear --python 3.14 "$release_dir/venv" -release_python="$release_dir/venv/bin/python" -uv pip install --python "$release_python" --constraint "$release_dir/constraints.txt" \ - dist/*.whl -uv pip check --python "$release_python" -"$release_python" -I - <<'PY' -from importlib.metadata import PackageNotFoundError, version +packages=(morphir morphir-python) +if [[ ${RELEASE_TAG:-} == morphir-v* ]]; then + packages=(morphir) +elif [[ ${RELEASE_TAG:-} == python-v* ]]; then + packages=(morphir-python) +fi + +for package in "${packages[@]}"; do + uv venv --clear --python 3.14 "$release_dir/$package-venv" + release_python="$release_dir/$package-venv/bin/python" + if [[ "$package" == morphir ]]; then + uv pip install --python "$release_python" packages/morphir/dist/*.whl + "$release_python" -I - <<'PY_CORE' +from importlib.metadata import distribution +from importlib.util import find_spec import morphir -import morphir_tools from morphir.ir import format_version from morphir.sdk import basics -assert morphir.__version__ == version("morphir-python") -assert morphir_tools.__version__ == version("morphir-python") +core = distribution("morphir") +assert morphir.__version__ == core.version +assert not core.requires +assert not core.entry_points assert basics.round(2.5) == 3 assert format_version.V3.major == 3 -try: - version("morphir") -except PackageNotFoundError: - pass -else: - raise AssertionError("Bundle verification must not install a separate morphir wheel") -PY -"$release_dir/venv/bin/morphir-python" --help -"$release_dir/venv/bin/morphir-python" ir inspect tests/fixtures/ir/v3/elm-simpleValueTree.json +assert find_spec("morphir_tools") is None +assert find_spec("typer") is None +PY_CORE + else + uv pip install --python "$release_python" --constraint "$release_dir/constraints.txt" \ + packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl + "$release_python" -I - <<'PY_CLI' +from importlib.metadata import distribution, version + +import morphir +import morphir_tools + +assert morphir.__version__ == version("morphir") +assert morphir_tools.__version__ == version("morphir-python") +core_files = set(distribution("morphir").files or []) +cli_files = set(distribution("morphir-python").files or []) +assert not core_files.intersection(cli_files) +PY_CLI + "$release_dir/$package-venv/bin/morphir-python" ir inspect \ + tests/fixtures/ir/v3/elm-simpleValueTree.json + fi + uv pip check --python "$release_python" + uv pip freeze --python "$release_python" --exclude morphir --exclude morphir-python \ + > "$release_dir/$package-requirements.txt" + if [[ -s "$release_dir/$package-requirements.txt" ]]; then + uvx --from pip-audit==2.10.1 pip-audit --strict --no-deps --disable-pip \ + --requirement "$release_dir/$package-requirements.txt" \ + --format json --output "$release_dir/$package-vulnerabilities.json" + else + echo '{"dependencies": [], "fixes": []}' > "$release_dir/$package-vulnerabilities.json" + echo "$package has no third-party runtime dependencies to audit" + fi + uvx --from pip-licenses==5.5.0 pip-licenses --python "$release_python" \ + --from=mixed \ + --allow-only='Apache-2.0;Apache Software License;MIT;MIT License;BSD-2-Clause;BSD-3-Clause;BSD License;ISC;ISC License;ISC License (ISCL)' \ + --format=json --output-file "$release_dir/$package-licenses.json" +done -# Audit the exact installed third-party versions without querying PyPI for the -# first-party bundle being released. -uv pip freeze --python "$release_python" --exclude morphir-python \ - > "$release_dir/requirements.txt" -uvx --from pip-audit==2.10.1 pip-audit --strict --no-deps --disable-pip \ - --requirement "$release_dir/requirements.txt" \ - --format json --output "$release_dir/vulnerabilities.json" +# Exercise the supported migration from the old distribution, which owned both +# namespaces. Uninstall old owners before installing the split distributions. +if [[ ${RELEASE_TAG:-} == morphir-v* ]]; then + exit 0 +fi +uv venv --clear --python 3.14 "$release_dir/migration-venv" +migration_python="$release_dir/migration-venv/bin/python" +uv pip install --python "$migration_python" --constraint "$release_dir/constraints.txt" \ + 'morphir-python==0.0.1' +uv pip uninstall --python "$migration_python" morphir-python morphir +uv pip install --python "$migration_python" --constraint "$release_dir/constraints.txt" \ + packages/morphir/dist/*.whl packages/morphir-tools/dist/*.whl +"$migration_python" -I - <<'PY_MIGRATION' +from importlib.metadata import version -# An exact allowlist rejects unknown licenses and newly introduced copyleft or -# proprietary licenses. Changes to this policy require maintainer review. -uvx --from pip-licenses==5.5.0 pip-licenses --python "$release_python" \ - --from=mixed \ - --allow-only='Apache-2.0;Apache Software License;MIT;MIT License;BSD-2-Clause;BSD-3-Clause;BSD License;ISC;ISC License;ISC License (ISCL)' \ - --format=json --output-file "$release_dir/licenses.json" +import morphir +import morphir_tools +from morphir.sdk import basics + +assert morphir.__version__ == version("morphir") +assert morphir_tools.__version__ == version("morphir-python") +assert basics.round(2.5) == 3 +PY_MIGRATION +"$release_dir/migration-venv/bin/morphir-python" --help +uv pip check --python "$migration_python" diff --git a/.config/mise/tasks/typecheck b/.config/mise/tasks/typecheck index 5e8fe8ce..b1903331 100755 --- a/.config/mise/tasks/typecheck +++ b/.config/mise/tasks/typecheck @@ -3,7 +3,7 @@ set -euo pipefail echo "Running mypy..." -uv run mypy packages/morphir/src packages/morphir-tools/src +uv run mypy packages/morphir/src packages/morphir-tools/src scripts echo "Running pyright..." uv run pyright packages/morphir/src packages/morphir-tools/src diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b088057b..fc5f63c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: workflow_call: inputs: release-tag: - description: Tag that must match the bundled package version + description: Package-specific tag that must match its project version type: string default: '' @@ -97,11 +97,18 @@ jobs: env: RELEASE_TAG: ${{ inputs.release-tag }} + - name: Upload morphir distributions + uses: actions/upload-artifact@v6 + with: + name: morphir-dist + path: packages/morphir/dist/ + if-no-files-found: error + - name: Upload morphir-python distributions uses: actions/upload-artifact@v6 with: name: morphir-python-dist - path: dist/ + path: packages/morphir-tools/dist/ if-no-files-found: error - name: Upload release scan reports diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 047c3d13..5118b2cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - 'v*' + - 'morphir-v*' - 'python-v*' # Qualification only. Publishing requires a tag push. workflow_dispatch: @@ -22,9 +22,35 @@ jobs: with: release-tag: ${{ github.ref_type == 'tag' && github.ref_name || '' }} + publish-morphir: + name: Publish morphir library and SDK to PyPI + if: >- + github.event_name == 'push' && + github.repository == 'finos/morphir-python' && + startsWith(github.ref, 'refs/tags/morphir-v') + needs: qualify + runs-on: ubuntu-latest + permissions: + id-token: write + environment: + name: pypi + url: https://pypi.org/project/morphir/ + steps: + - name: Download qualified morphir distributions + uses: actions/download-artifact@v7 + with: + name: morphir-dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + publish-morphir-python: name: Publish morphir-python to PyPI - if: github.event_name == 'push' && github.repository == 'finos/morphir-python' + if: >- + github.event_name == 'push' && + github.repository == 'finos/morphir-python' && + startsWith(github.ref, 'refs/tags/python-v') needs: qualify runs-on: ubuntu-latest permissions: @@ -44,7 +70,12 @@ jobs: github-release: name: Create GitHub Release - needs: publish-morphir-python + needs: [publish-morphir, publish-morphir-python] + if: >- + always() && !cancelled() && + (needs.publish-morphir.result == 'success' || + needs.publish-morphir-python.result == 'success') && + !contains(needs.*.result, 'failure') runs-on: ubuntu-latest permissions: contents: write @@ -52,10 +83,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Download all artifacts + - name: Download the published package artifacts uses: actions/download-artifact@v7 with: - pattern: '*-dist' + name: ${{ startsWith(github.ref, 'refs/tags/morphir-v') && 'morphir-dist' || 'morphir-python-dist' }} path: dist/ merge-multiple: true diff --git a/AGENTS.md b/AGENTS.md index e957b365..2ec26cc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Morphir Python is a port of Morphir to Python, providing functional domain model morphir-python/ ├── packages/ │ ├── morphir/ # Core library workspace - IR models, types, pure functions -│ └── morphir-tools/ # CLI source; root morphir-python bundles library and CLI +│ └── morphir-tools/ # morphir-python CLI distribution (depends on morphir) ├── tests/ │ ├── unit/ # Unit tests for morphir package │ └── tools/ # Unit tests for morphir-python package @@ -21,10 +21,12 @@ morphir-python/ ### Package Responsibilities -The root `pyproject.toml` publishes one `morphir-python` distribution containing -both source components. It provides `import morphir`, `import morphir_tools`, and -the `morphir-python` command. The core remains a standalone workspace for development; -the release workflow does not publish or require a separate `morphir` PyPI package. +The root `pyproject.toml` is a development workspace. `packages/morphir/` publishes +`morphir`, containing the library and SDK without third-party runtime dependencies. +`packages/morphir-tools/` publishes `morphir-python`, containing the CLI and depending +on `morphir`. They own separate import namespaces and have independent versions and +release tags (`morphir-v*` and `python-v*`). The CLI is the future integration surface +for MEP extension-protocol commands and potential WASM build tooling. **`morphir` (core library)** - Morphir IR (Intermediate Representation) models, in two versions: diff --git a/README.md b/README.md index ac378271..79f93fd4 100644 --- a/README.md +++ b/README.md @@ -9,24 +9,47 @@ Morphir enables you to write business logic once and use it across multiple plat ## Installation -Install the library, Morphir SDK runtime, and CLI together: +The library and CLI are separate distributions: + +| PyPI distribution | Contents | Python import / command | +|-------------------|----------|-------------------------| +| [`morphir`](https://pypi.org/project/morphir/) | Core library, IR models, and SDK runtime; no third-party runtime dependencies | `import morphir` | +| [`morphir-python`](https://pypi.org/project/morphir-python/) | CLI and tool integrations; depends on `morphir` | `import morphir_tools`; `morphir-python` | + +Install just the library and SDK: ```bash -pip install morphir-python +pip install morphir==0.0.1 +# or: uv add morphir==0.0.1 ``` -Or with uv: +Install the CLI, which also installs the library: ```bash -uv add morphir-python +pip install morphir-python==0.0.2 +# or: uv add morphir-python==0.0.2 ``` -| PyPI distribution | Python imports | Command | -|-------------------|----------------|---------| -| [`morphir-python`](https://pypi.org/project/morphir-python/) | `morphir`, `morphir_tools` | `morphir-python` | +Python 3.14 or later is required. The CLI currently provides IR inspection and +conversion. It is also the intended home for future MEP extension-protocol CLI +integration and possible WASM build tooling; those features are not implemented +by this packaging change. + +### Migrating from the bundled 0.0.1 release -The distribution includes the core library and tools. It does not require a -separate `morphir` package from PyPI. +`morphir-python==0.0.1` bundled both namespaces. Starting with `0.0.2`, library +files belong to the separate `morphir` distribution. Use a fresh virtual +environment, or remove both old package owners before reinstalling in your +existing virtual environment: + +```bash +python -m pip uninstall -y morphir-python morphir +python -m pip install morphir-python==0.0.2 +``` + +Library-only consumers can install `morphir==0.0.1` instead. Imports remain +`morphir` and `morphir_tools`; the command remains `morphir-python`. The release +checks exercise the uninstall/reinstall migration from the published old bundle. ## Quick Start @@ -148,52 +171,71 @@ mise run check ## Releases -The initial PyPI release is `morphir-python==0.0.1`, containing both the `morphir` -library and the `morphir-python` CLI. Python 3.14 or later is required. The root -`pyproject.toml` defines the public distribution. The library workspace under -`packages/morphir/` remains independent of CLI dependencies for development; it -is included in the bundle rather than published separately by this workflow. - -Publishing uses [PyPI trusted publishing](https://docs.pypi.org/trusted-publishers/using-a-publisher/) -without a stored PyPI token. Configure the `morphir-python` project on PyPI with: - -| Publisher field | Value | -|-----------------|-------| -| Owner | `finos` | -| Repository | `morphir-python` | -| Workflow filename | `release.yml` | -| Environment | `pypi` | - -The GitHub `pypi` environment permits release tags matching `v*` or `python-v*`. -After maintainer approval, push a tag such as `python-v0.0.1` whose version matches -root package metadata and the runtime `__version__` values. Refresh `uv.lock` -when changing versions. A new tag records the corrected source without moving -the earlier failed `v0.0.1` tag. - -The release workflow runs the same lint, format, type, coverage and BDD checks as -PR CI. It builds the bundle once, validates wheel and source metadata and license -files, verifies that every library and CLI source file is in the wheel, installs -only that wheel in an isolated environment, checks imports and the CLI, then -scans the locked runtime dependency versions with `pip-audit` and `pip-licenses`. -Known vulnerabilities, audit failures, unknown licenses and licenses outside the -Apache-2.0, MIT, BSD-2-Clause, BSD-3-Clause and ISC allowlist block publication. -The allowlist also accepts corresponding legacy license classifiers. The CVE -scan queries third-party dependencies, excluding the first-party bundle being -released. These checks use package metadata and known vulnerability advisories; -they do not replace a source-code license or security review. Scan reports are -retained as workflow artifacts. - -Only the publishing job receives `id-token: write`. It publishes the checked -artifacts, then a separate job creates the GitHub release. The workflow's manual -**Run workflow** action runs qualification without publishing. To qualify locally: - -```bash -mise run install -mise run check -mise run coverage -mise run build -RELEASE_TAG=python-v0.0.1 mise run release-check -``` +The repository root is a development workspace. Public package metadata is in +`packages/morphir/pyproject.toml` and `packages/morphir-tools/pyproject.toml`. +Versions are independent: the first library release is `morphir==0.0.1`, and the +first CLI-only release is `morphir-python==0.0.2`. The already published bundled +CLI `0.0.1` and its `python-v0.0.1` tag remain unchanged. + +### Trusted publisher configuration + +Configure a [trusted publisher](https://docs.pypi.org/trusted-publishers/using-a-publisher/) +for **each** PyPI project. Use a pending publisher if a project has no release yet. +Both publishers use this repository and the same workflow/environment identity: + +| Field | Library / SDK | CLI | +|-------|---------------|-----| +| PyPI project | `morphir` | `morphir-python` | +| Repository owner | `finos` | `finos` | +| Repository name | `morphir-python` | `morphir-python` | +| Workflow filename | `release.yml` | `release.yml` | +| Environment | `pypi` | `pypi` | +| Tag example | `morphir-v0.0.1` | `python-v0.0.2` | + +On GitHub, the `pypi` environment permits both `morphir-v*` and `python-v*` tag +patterns. No stored PyPI token is needed. Only the selected publishing job +receives `id-token: write`; a manual workflow run qualifies artifacts without +publishing. Bare `v*` tags are not used for new releases. + +### Qualification and publication + +1. Update the selected package's metadata and runtime `__version__`. Update the + CLI's minimum library dependency when its API requirements change, and run + `uv lock`. +2. Run `mise run install`, `mise run check`, `mise run coverage`, `mise run build` + and `mise run release-check`. Without a tag, this qualifies both distributions. +3. After maintainer approval, tag the candidate with the selected package prefix + and exact version, then push that tag. To release the initial library: + + ```bash + git tag -a morphir-v0.0.1 -m 'Release morphir library and SDK 0.0.1' + git push origin morphir-v0.0.1 + ``` + +4. Publish the library before any CLI that needs it. A later `python-v0.0.2` tag + publishes only the CLI. Its qualification requires the published library wheel + to have the same SHA256 digest as the core artifact used in testing. If core + sources or metadata change, bump and publish the core version first. +5. Verify the selected PyPI files and a clean installation. The workflow creates + a GitHub release containing only the selected package's artifacts. Never move + an existing release tag or overwrite an existing PyPI version. + +Release and PR CI run lint, formatting, type, coverage and BDD checks before +building both candidates. Artifact checks enforce exact names/versions, complete source inclusion, +separate namespace ownership, Apache license files, no core runtime dependencies +or console scripts, and the CLI's versioned core dependency. Isolated installs +exercise the library/SDK without CLI dependencies, CLI behavior, and migration +from the old bundle. PR qualification checks both runtime environments; a tagged +release scans only its selected package, and the migration check runs for CLI +qualification. CVE and license checks cover locked installed runtime versions; +the core has no third-party dependencies. Scan reports are retained as +workflow artifacts. Scans use metadata/advisories and do not replace source-code +security or license review. + +For targeted qualification, `RELEASE_TAG=morphir-v0.0.1 mise run release-check` +validates the library tag. `RELEASE_TAG=python-v0.0.2 mise run release-check` also +checks the published core dependency. The latter must wait until the library is +published. Unknown tags or a tag/version mismatch block publication. ## Project Principles diff --git a/packages/morphir-tools/README.md b/packages/morphir-tools/README.md index 66b8a68d..041b8425 100644 --- a/packages/morphir-tools/README.md +++ b/packages/morphir-tools/README.md @@ -21,9 +21,10 @@ Or with uv: uv add morphir-python ``` -The `morphir-python` distribution includes the `morphir` core library and SDK -runtime alongside the CLI. It does not depend on a separate `morphir` PyPI -package. Python imports remain `morphir` and `morphir_tools`. +The `morphir-python` distribution contains the CLI and depends on the separate +`morphir` library/SDK distribution. Python imports remain `morphir` and +`morphir_tools`. Version 0.0.1 was a bundle; for migration to CLI-only 0.0.2, see +the [repository installation guide](../../README.md#migrating-from-the-bundled-001-release). ## Usage @@ -58,7 +59,7 @@ format-version rules and the draft v4 status. ## Requirements - Python 3.14+ -- Core library and SDK included +- `morphir>=0.0.1` (installed as a dependency) ## License diff --git a/packages/morphir-tools/pyproject.toml b/packages/morphir-tools/pyproject.toml new file mode 100644 index 00000000..3cbefe69 --- /dev/null +++ b/packages/morphir-tools/pyproject.toml @@ -0,0 +1,45 @@ +[project] +name = "morphir-python" +version = "0.0.2" +description = "CLI tools and extensions for Morphir Python" +readme = "README.md" +license = "Apache-2.0" +license-files = ["LICENSE"] +requires-python = ">=3.14" +keywords = ["morphir", "cli", "tools", "code-generation", "domain-modeling"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Libraries", + "Typing :: Typed", +] +authors = [ + { name = "FINOS", email = "morphir@finos.org" } +] +dependencies = [ + "morphir>=0.0.1", + "typer>=0.15", + # Rich uses Pygments; older versions are affected by CVE-2026-4539. + "pygments>=2.20.0", +] + +[project.urls] +Homepage = "https://github.com/finos/morphir-python" +Documentation = "https://morphir.finos.org" +Repository = "https://github.com/finos/morphir-python" +Issues = "https://github.com/finos/morphir-python/issues" + +[project.scripts] +morphir-python = "morphir_tools.cli:main" + +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/morphir_tools"] diff --git a/packages/morphir-tools/src/morphir_tools/__init__.py b/packages/morphir-tools/src/morphir_tools/__init__.py index 7d28926c..67c5d38b 100644 --- a/packages/morphir-tools/src/morphir_tools/__init__.py +++ b/packages/morphir-tools/src/morphir_tools/__init__.py @@ -6,8 +6,8 @@ Example: >>> import morphir_tools >>> morphir_tools.__version__ - '0.0.1' + '0.0.2' """ -__version__ = "0.0.1" +__version__ = "0.0.2" __all__ = ["__version__"] diff --git a/packages/morphir/README.md b/packages/morphir/README.md index 233347f0..faaa6c4e 100644 --- a/packages/morphir/README.md +++ b/packages/morphir/README.md @@ -11,17 +11,18 @@ This is the core library package containing: ## Installation -The core library is included in the `morphir-python` distribution, together with -the SDK runtime and CLI. Python code continues to use `import morphir`. +The `morphir` distribution contains the library and SDK with no third-party +runtime dependencies. Python code uses `import morphir`. Install `morphir-python` +separately if you need the CLI. ```bash -pip install morphir-python +pip install morphir ``` Or with uv: ```bash -uv add morphir-python +uv add morphir ``` ## Usage diff --git a/pyproject.toml b/pyproject.toml index b1d35983..a2e7a866 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,45 +1,10 @@ [project] -name = "morphir-python" +name = "morphir-workspace" version = "0.0.1" -description = "Morphir domain modeling library, SDK runtime, and CLI for Python" +description = "Development workspace for the Morphir library and CLI" requires-python = ">=3.14" readme = "README.md" license = "Apache-2.0" -license-files = ["LICENSE"] -authors = [{ name = "FINOS", email = "morphir@finos.org" }] -keywords = ["morphir", "domain-modeling", "functional", "cli", "ir"] -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.14", - "Typing :: Typed", -] -dependencies = [ - "typer>=0.15", - # Rich uses Pygments; older versions are affected by CVE-2026-4539. - "pygments>=2.20.0", -] - -[project.urls] -Homepage = "https://github.com/finos/morphir-python" -Documentation = "https://morphir.finos.org" -Repository = "https://github.com/finos/morphir-python" -Issues = "https://github.com/finos/morphir-python/issues" - -[project.scripts] -morphir-python = "morphir_tools.cli:main" - -[build-system] -requires = ["hatchling>=1.27"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["packages/morphir/src/morphir", "packages/morphir-tools/src/morphir_tools"] - -[tool.hatch.build.targets.sdist] -only-include = ["packages", "README.md", "LICENSE", "uv.lock"] [dependency-groups] dev = [ @@ -59,7 +24,10 @@ test = [ managed = true [tool.uv.workspace] -members = ["packages/morphir"] +members = ["packages/morphir", "packages/morphir-tools"] + +[tool.uv.sources] +morphir = { workspace = true } # ============================================================================= # Ruff Configuration @@ -136,13 +104,14 @@ pythonVersion = "3.14" typeCheckingMode = "standard" reportMissingImports = true reportMissingTypeStubs = false -include = ["packages/morphir/src", "packages/morphir-tools/src", "tests"] +include = ["packages/morphir/src", "packages/morphir-tools/src", "scripts", "tests"] exclude = ["**/__pycache__", ".venv", "build", "dist"] # ============================================================================= # Pytest Configuration # ============================================================================= [tool.pytest.ini_options] +pythonpath = ["."] testpaths = ["tests"] python_files = ["test_*.py"] python_classes = ["Test*"] diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e7d2ac04 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Repository build and release utilities.""" diff --git a/scripts/check_distributions.py b/scripts/check_distributions.py new file mode 100644 index 00000000..0b25a5e8 --- /dev/null +++ b/scripts/check_distributions.py @@ -0,0 +1,171 @@ +"""Validate independently versioned library and CLI release artifacts.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tarfile +import tomllib +import zipfile +from configparser import ConfigParser +from dataclasses import dataclass +from email.parser import BytesParser +from pathlib import Path +from typing import TYPE_CHECKING +from urllib.request import urlopen + +if TYPE_CHECKING: + from collections.abc import Mapping + from email.message import Message + + +@dataclass(frozen=True) +class ReleasePackage: + """A distribution and the source namespace it exclusively owns.""" + + name: str + version: str + namespace: str + directory: Path + tag_prefix: str + + @property + def stem(self) -> str: + """Return the normalized archive name and version.""" + return f"{self.name.replace('-', '_')}-{self.version}" + + @property + def wheel(self) -> Path: + """Return the pure Python wheel path.""" + return self.directory / "dist" / f"{self.stem}-py3-none-any.whl" + + +def load_packages(root: Path) -> tuple[ReleasePackage, ...]: + """Read independent versions from the two public package projects.""" + packages: list[ReleasePackage] = [] + for name, folder, namespace, prefix in ( + ("morphir", "morphir", "morphir", "morphir-v"), + ("morphir-python", "morphir-tools", "morphir_tools", "python-v"), + ): + directory = root / "packages" / folder + project = tomllib.loads((directory / "pyproject.toml").read_text())["project"] + if project["name"] != name: + raise ValueError(f"Unexpected project identity in {directory}") + packages.append( + ReleasePackage(name, project["version"], namespace, directory, prefix) + ) + return tuple(packages) + + +def select_package( + tag: str, packages: tuple[ReleasePackage, ...] +) -> ReleasePackage | None: + """Select only the package whose prefix and exact version match the tag.""" + if not tag: + return None + for package in packages: + if tag == f"{package.tag_prefix}{package.version}": + return package + raise ValueError(f"Invalid release tag {tag!r}: expected a package prefix/version") + + +def validate_metadata(package: ReleasePackage, metadata: Message) -> None: + """Enforce identity, licensing and the library-to-CLI dependency boundary.""" + if metadata["Name"] != package.name or metadata["Version"] != package.version: + raise ValueError(f"Incorrect distribution identity for {package.name}") + if metadata["License-Expression"] != "Apache-2.0": + raise ValueError(f"Missing Apache-2.0 license for {package.name}") + requirements = metadata.get_all("Requires-Dist", []) + if package.name == "morphir" and requirements: + raise ValueError("morphir must have no runtime dependencies") + if package.name == "morphir-python" and not any( + requirement.startswith("morphir>=") for requirement in requirements + ): + raise ValueError("morphir-python must declare a versioned library dependency") + + +def validate_archive(package: ReleasePackage, path: Path) -> None: + """Check archive metadata, namespace ownership, source completeness and CLI.""" + if path.suffix == ".whl": + with zipfile.ZipFile(path) as archive: + names = set(archive.namelist()) + raw = archive.read(f"{package.stem}.dist-info/METADATA") + license_path = f"{package.stem}.dist-info/licenses/LICENSE" + entry_points = f"{package.stem}.dist-info/entry_points.txt" + if package.name == "morphir" and entry_points in names: + raise ValueError("The library must not install console commands") + if package.name == "morphir-python": + config = ConfigParser() + config.read_string(archive.read(entry_points).decode()) + if dict(config["console_scripts"]) != { + "morphir-python": "morphir_tools.cli:main" + }: + raise ValueError("Incorrect morphir-python console command") + source_prefix = "" + else: + with tarfile.open(path) as archive: + names = set(archive.getnames()) + member = archive.extractfile(f"{package.stem}/PKG-INFO") + if member is None: + raise ValueError(f"Missing package metadata in {path}") + raw = member.read() + license_path = f"{package.stem}/LICENSE" + source_prefix = f"{package.stem}/src/" + validate_metadata(package, BytesParser().parsebytes(raw)) + if license_path not in names: + raise ValueError(f"Missing license file in {path}") + other = "morphir_tools" if package.name == "morphir" else "morphir" + if any(name.startswith(f"{source_prefix}{other}/") for name in names): + raise ValueError(f"Unexpected bundled namespace {other} in {path}") + source = package.directory / "src" + expected = { + source_prefix + p.relative_to(source).as_posix() + for p in (source / package.namespace).rglob("*") + if p.is_file() and p.suffix in (".py", ".typed") + } + missing = expected - names + if not expected or missing: + raise ValueError(f"Missing package sources in {path}: {missing}") + + +def match_published_wheel(wheel: Path, published: Mapping[str, str]) -> None: + """Refuse CLI promotion if its tested core differs from the published core.""" + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + if published.get(wheel.name) != digest: + raise ValueError( + "The published library must match the qualified core wheel; " + "publish a new library version before releasing this CLI" + ) + + +def main() -> None: + """Validate both candidates and require the selected CLI's published core.""" + packages = load_packages(Path(__file__).resolve().parents[1]) + selected = select_package(os.environ.get("RELEASE_TAG", ""), packages) + for package in packages: + expected = {package.wheel.name, f"{package.stem}.tar.gz"} + actual = { + p.name + for p in (package.directory / "dist").iterdir() + if p.is_file() and p.name != ".gitignore" + } + if actual != expected: + raise ValueError(f"Unexpected distributions for {package.name}: {actual}") + for filename in expected: + validate_archive(package, package.directory / "dist" / filename) + print(f"Validated {package.name} {package.version}") + if selected is not None and selected.name == "morphir-python": + core = packages[0] + url = f"https://pypi.org/pypi/morphir/{core.version}/json" + with urlopen(url, timeout=30) as response: + data = json.load(response) + digests: dict[str, str] = { + item["filename"]: item["digests"]["sha256"] for item in data["urls"] + } + match_published_wheel(core.wheel, digests) + print("The CLI's qualified core matches the published library") + + +if __name__ == "__main__": + main() diff --git a/tests/tools/test_tools.py b/tests/tools/test_tools.py index c7ffdfc3..bf035840 100644 --- a/tests/tools/test_tools.py +++ b/tests/tools/test_tools.py @@ -12,9 +12,9 @@ def test_version_is_defined(self) -> None: """The package should have a version attribute.""" assert hasattr(morphir_tools, "__version__") - def test_version_matches_expected(self, sample_version: str) -> None: + def test_version_matches_expected(self) -> None: """The version should match the expected value.""" - assert morphir_tools.__version__ == sample_version + assert morphir_tools.__version__ == "0.0.2" def test_installed_distribution_name(self) -> None: """The tools import belongs to the morphir-python distribution.""" @@ -29,10 +29,10 @@ def test_installed_console_command(self) -> None: } assert commands == {"morphir-python": "morphir_tools.cli:main"} - def test_bundle_does_not_require_separate_core(self) -> None: - """Installing the bundle must not query PyPI for an unpublished core.""" + def test_cli_requires_separate_core(self) -> None: + """The CLI must install the independent library and SDK distribution.""" requirements = distribution("morphir-python").requires or [] - assert not any(item.startswith("morphir") for item in requirements) + assert "morphir>=0.0.1" in requirements def test_version_is_string(self) -> None: """The version should be a string.""" diff --git a/tests/unit/test_morphir.py b/tests/unit/test_morphir.py index a8401d9d..2cf08234 100644 --- a/tests/unit/test_morphir.py +++ b/tests/unit/test_morphir.py @@ -1,5 +1,7 @@ """Tests for the morphir core library package.""" +from importlib.metadata import distribution + import morphir @@ -10,6 +12,12 @@ def test_version_is_defined(self) -> None: """The package should have a version attribute.""" assert hasattr(morphir, "__version__") + def test_core_has_no_runtime_dependencies_or_commands(self) -> None: + """Library-only installs must not pull in CLI dependencies or scripts.""" + package = distribution("morphir") + assert not package.requires + assert not package.entry_points + def test_version_matches_expected(self, sample_version: str) -> None: """The version should match the expected value.""" assert morphir.__version__ == sample_version diff --git a/tests/unit/test_release_process.py b/tests/unit/test_release_process.py new file mode 100644 index 00000000..b764bd4f --- /dev/null +++ b/tests/unit/test_release_process.py @@ -0,0 +1,104 @@ +"""Release selection and package ownership checks.""" + +import hashlib +import zipfile +from email.message import Message +from pathlib import Path + +import pytest + +from scripts.check_distributions import ( + ReleasePackage, + match_published_wheel, + select_package, + validate_archive, + validate_metadata, +) + +CORE = ReleasePackage("morphir", "0.0.1", "morphir", Path("core"), "morphir-v") +CLI = ReleasePackage( + "morphir-python", "0.0.2", "morphir_tools", Path("cli"), "python-v" +) + + +@pytest.mark.parametrize( + ("tag", "expected"), + [("", None), ("morphir-v0.0.1", CORE), ("python-v0.0.2", CLI)], +) +def test_independent_release_versions( + tag: str, expected: ReleasePackage | None +) -> None: + assert select_package(tag, (CORE, CLI)) == expected + + +@pytest.mark.parametrize("tag", ["v0.0.1", "morphir-v0.0.2", "python-v0.0.1"]) +def test_wrong_or_ambiguous_release_tag_is_rejected(tag: str) -> None: + with pytest.raises(ValueError, match="release tag"): + select_package(tag, (CORE, CLI)) + + +def metadata(package: ReleasePackage) -> Message: + result = Message() + result["Name"] = package.name + result["Version"] = package.version + result["License-Expression"] = "Apache-2.0" + return result + + +def test_library_rejects_cli_dependencies() -> None: + value = metadata(CORE) + validate_metadata(CORE, value) + value["Requires-Dist"] = "typer>=0.15" + with pytest.raises(ValueError, match="no runtime dependencies"): + validate_metadata(CORE, value) + + +def test_cli_requires_a_versioned_library_dependency() -> None: + value = metadata(CLI) + with pytest.raises(ValueError, match="library dependency"): + validate_metadata(CLI, value) + value["Requires-Dist"] = "morphir>=0.0.1" + validate_metadata(CLI, value) + + +def test_wrong_distribution_identity_is_rejected() -> None: + value = metadata(CORE) + value.replace_header("Name", "morphir-python") + with pytest.raises(ValueError, match="identity"): + validate_metadata(CORE, value) + + +def test_cli_promotion_rejects_a_different_published_library(tmp_path: Path) -> None: + wheel = tmp_path / "morphir-0.0.1-py3-none-any.whl" + wheel.write_bytes(b"tested core candidate") + with pytest.raises(ValueError, match="published library"): + match_published_wheel(wheel, {wheel.name: "different-digest"}) + with pytest.raises(ValueError, match="published library"): + match_published_wheel(wheel, {}) + match_published_wheel( + wheel, {wheel.name: hashlib.sha256(b"tested core candidate").hexdigest()} + ) + + +@pytest.mark.parametrize("name", ["morphir", "morphir-python"]) +def test_archives_reject_the_other_packages_namespace( + tmp_path: Path, name: str +) -> None: + namespace = "morphir" if name == "morphir" else "morphir_tools" + other = "morphir_tools" if name == "morphir" else "morphir" + package = ReleasePackage(name, "0.0.1", namespace, tmp_path, "unused-v") + value = metadata(package) + if name == "morphir-python": + value["Requires-Dist"] = "morphir>=0.0.1" + path = tmp_path / f"{package.stem}-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(f"{package.stem}.dist-info/METADATA", value.as_bytes()) + archive.writestr(f"{package.stem}.dist-info/licenses/LICENSE", "Apache-2.0") + archive.writestr(f"{other}/__init__.py", "") + if name == "morphir-python": + archive.writestr( + f"{package.stem}.dist-info/entry_points.txt", + "[console_scripts]\nmorphir-python = morphir_tools.cli:main\n", + ) + with pytest.raises(ValueError, match="Unexpected bundled namespace"): + validate_archive(package, path) diff --git a/uv.lock b/uv.lock index e57b1709..84da297b 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,7 @@ requires-python = ">=3.14" members = [ "morphir", "morphir-python", + "morphir-workspace", ] [[package]] @@ -199,13 +200,26 @@ source = { editable = "packages/morphir" } [[package]] name = "morphir-python" -version = "0.0.1" -source = { editable = "." } +version = "0.0.2" +source = { editable = "packages/morphir-tools" } dependencies = [ + { name = "morphir" }, { name = "pygments" }, { name = "typer" }, ] +[package.metadata] +requires-dist = [ + { name = "morphir", editable = "packages/morphir" }, + { name = "pygments", specifier = ">=2.20.0" }, + { name = "typer", specifier = ">=0.15" }, +] + +[[package]] +name = "morphir-workspace" +version = "0.0.1" +source = { virtual = "." } + [package.dev-dependencies] dev = [ { name = "mypy" }, @@ -221,10 +235,6 @@ test = [ ] [package.metadata] -requires-dist = [ - { name = "pygments", specifier = ">=2.20.0" }, - { name = "typer", specifier = ">=0.15" }, -] [package.metadata.requires-dev] dev = [