diff --git a/.config/mise/tasks/docs/test b/.config/mise/tasks/docs/test new file mode 100755 index 00000000..d3ae7631 --- /dev/null +++ b/.config/mise/tasks/docs/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +#MISE description="Check canonical documentation snippets, installed examples and types" +set -euo pipefail +uv run --no-sync python scripts/docs_examples.py check +uv run --no-sync pytest tests/tools/test_python_docs.py tests/unit/test_docs_examples.py diff --git a/.config/mise/tasks/site/build b/.config/mise/tasks/site/build new file mode 100755 index 00000000..8b9e83bd --- /dev/null +++ b/.config/mise/tasks/site/build @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Build the Telescope documentation site" +set -euo pipefail +exec python3 scripts/site_tool.py build "$@" diff --git a/.config/mise/tasks/site/check b/.config/mise/tasks/site/check new file mode 100755 index 00000000..d565ee56 --- /dev/null +++ b/.config/mise/tasks/site/check @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Check the Telescope documentation site" +set -euo pipefail +exec python3 scripts/site_tool.py check "$@" diff --git a/.config/mise/tasks/site/serve b/.config/mise/tasks/site/serve new file mode 100755 index 00000000..389b79b5 --- /dev/null +++ b/.config/mise/tasks/site/serve @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Serve the Telescope documentation site" +set -euo pipefail +exec python3 scripts/site_tool.py serve "$@" diff --git a/.config/mise/tasks/site/setup b/.config/mise/tasks/site/setup new file mode 100755 index 00000000..fc85cb5c --- /dev/null +++ b/.config/mise/tasks/site/setup @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +#MISE description="Setup the Telescope documentation site" +set -euo pipefail +exec python3 scripts/site_tool.py setup "$@" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a540ff8..28dfb8ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: workflow_call: inputs: release-tag: @@ -27,6 +28,8 @@ jobs: outputs: jobs: ${{ steps.select.outputs.jobs }} reason: ${{ steps.select.outputs.reason }} + docs-examples: ${{ steps.select.outputs.docs-examples }} + docs-hosts: ${{ steps.select.outputs.docs-hosts }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -285,10 +288,23 @@ jobs: .dev/qualification/*.json include-hidden-files: true + docs: + name: Documentation + needs: [changes, native-wheel] + if: >- + always() && needs.changes.result == 'success' && + fromJSON(needs.changes.outputs.jobs).docs && + (needs.native-wheel.result == 'success' || needs.native-wheel.result == 'skipped') + uses: ./.github/workflows/docs.yml + with: + examples: ${{ needs.changes.outputs.docs-examples == 'true' }} + hosts: ${{ needs.changes.outputs.docs-hosts == 'true' }} + native-wheel: ${{ fromJSON(needs.changes.outputs.jobs)['native-wheel'] }} + all-checks: name: All Checks if: always() - needs: [changes, lint, typecheck, native-wheel, test, rust, bundle, build] + needs: [changes, lint, typecheck, native-wheel, test, rust, bundle, build, docs] runs-on: ubuntu-latest steps: - name: Checkout repository diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..05c77b90 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,154 @@ +name: Documentation + +on: + workflow_call: + inputs: + examples: + type: boolean + default: false + hosts: + type: boolean + default: false + native-wheel: + type: boolean + default: false + +permissions: + contents: read + +jobs: + examples: + name: Installed documentation examples + if: inputs.examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - uses: astral-sh/setup-uv@v7 + - name: Reuse the selected native producer + if: inputs.native-wheel + uses: actions/download-artifact@v7 + with: + name: native-linux-x86_64 + path: dist/native + - name: Verify released, candidate and library-only examples + env: + NATIVE_WHEEL: ${{ inputs.native-wheel }} + run: | + args=() + if [[ "$NATIVE_WHEEL" == true ]]; then args+=(--native-wheel-dir dist/native); fi + python scripts/docs_check.py "${args[@]}" + - uses: actions/upload-artifact@v6 + with: + name: documentation-examples + path: .dev/docs-examples.json + include-hidden-files: true + if-no-files-found: error + + hosts: + name: Host recipe qualification + if: inputs.hosts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - uses: actions/setup-node@v6 + with: + node-version: '24' + - uses: astral-sh/setup-uv@v7 + - uses: jdx/mise-action@v3 + with: + install: false + - name: Install the qualified Python pair + run: | + uv venv .dev/docs-host-python + uv pip install --python .dev/docs-host-python/bin/python --only-binary=:all: --refresh morphir==0.1.0 morphir-python==0.1.0 + - name: Run displayed recipe, offline bundle checks and MoonBit refusal probes + run: .dev/docs-host-python/bin/python scripts/docs_hosts.py qualify + - uses: actions/upload-artifact@v6 + with: + name: documentation-hosts + path: .dev/docs-hosts/report.json + include-hidden-files: true + if-no-files-found: error + + site: + name: Site build, checks and browser smoke + needs: [examples, hosts] + if: >- + always() && + (!inputs.examples || needs.examples.result == 'success') && + (!inputs.hosts || needs.hosts.result == 'success') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: '3.14' + - uses: actions/setup-node@v6 + with: + node-version: '24' + - uses: astral-sh/setup-uv@v7 + - name: Check site tooling and failure behavior + env: + MORPHIR_SITE_TEST: '1' + run: | + uv venv .dev/site-checker + uv pip install --python .dev/site-checker/bin/python pytest==9.0.2 ruff==0.14.13 + .dev/site-checker/bin/ruff check scripts/docs_*.py scripts/site_*.py tests/unit/test_docs*.py tests/unit/test_site*.py + .dev/site-checker/bin/ruff format --check scripts/docs_*.py scripts/site_*.py tests/unit/test_docs*.py tests/unit/test_site*.py + .dev/site-checker/bin/python -m pytest tests/unit/test_site_tool.py tests/unit/test_site_release.py tests/unit/test_docs_hosts.py tests/unit/test_docs_examples.py tests/unit/test_ci_changes.py + - name: Install browser checks + run: | + npm ci --prefix site/browser + npm exec --prefix site/browser -- playwright install --with-deps chromium + - name: Build once at each base and check the actual browser experience + run: | + npm test --prefix site/browser + python scripts/site_tool.py check + python scripts/site_tool.py check --base / --out .dev/site-root + - name: Download example evidence + if: inputs.examples + uses: actions/download-artifact@v7 + with: + name: documentation-examples + path: .dev/site-evidence + - name: Download host evidence + if: inputs.hosts + uses: actions/download-artifact@v7 + with: + name: documentation-hosts + path: .dev/site-evidence + - name: Bind checked bytes and required evidence + env: + EXAMPLES: ${{ inputs.examples }} + HOSTS: ${{ inputs.hosts }} + run: | + args=() + if [[ "$EXAMPLES" == true ]]; then args+=(--examples .dev/site-evidence/docs-examples.json); fi + if [[ "$HOSTS" == true ]]; then + args+=(--hosts .dev/site-evidence/report.json) + else + args+=(--hosts site/evidence/hosts.json) + fi + python scripts/site_release.py seal "${args[@]}" + python scripts/site_tool.py check + - name: Retain the exact verified output + uses: actions/upload-artifact@v6 + with: + name: documentation-site + path: _site/ + if-no-files-found: error + retention-days: 30 + - name: Retain browser evidence + if: always() + uses: actions/upload-artifact@v6 + with: + name: documentation-browser + path: .dev/site-browser/ + include-hidden-files: true + retention-days: 30 diff --git a/.github/workflows/site-publish.yml b/.github/workflows/site-publish.yml new file mode 100644 index 00000000..ca9a0ad2 --- /dev/null +++ b/.github/workflows/site-publish.yml @@ -0,0 +1,73 @@ +name: Publish documentation + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + run-id: + description: Successful main CI run whose retained checked site should be deployed or restored + required: true + type: string + +permissions: + contents: read + actions: read + +concurrency: + group: documentation-pages + cancel-in-progress: false + +jobs: + publish: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch')) + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + permissions: + contents: read + actions: read + pages: write + id-token: write + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Validate the producer before consuming its artifact + id: producer + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id || inputs.run-id }} + REPOSITORY: ${{ github.repository }} + run: | + [[ "$RUN_ID" =~ ^[0-9]+$ ]] + mkdir -p .dev + gh api "repos/$REPOSITORY/actions/runs/$RUN_ID" > .dev/site-producer.json + python3 scripts/site_release.py producer --run-json .dev/site-producer.json --repository "$REPOSITORY" >> "$GITHUB_OUTPUT" + - name: Checkout the producer's exact source + uses: actions/checkout@v6 + with: + ref: ${{ steps.producer.outputs.source }} + path: candidate + persist-credentials: false + - uses: actions/download-artifact@v7 + with: + name: documentation-site + path: _site + run-id: ${{ steps.producer.outputs.run }} + github-token: ${{ github.token }} + - name: Verify checked bytes and candidate identity without rebuilding + env: + SOURCE_SHA: ${{ steps.producer.outputs.source }} + run: python3 candidate/scripts/site_release.py verify --site _site --source "$SOURCE_SHA" --deploy + - uses: actions/upload-pages-artifact@v4 + with: + path: _site + - id: deploy + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 1d6b575f..7e0eeb03 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,8 @@ rust/target/ # Extension bundles built by mise run extension:bundle .morphir/ + +# Checked documentation output and browser dependencies +_site/ +site/browser/node_modules/ +site/downloads/ diff --git a/README.md b/README.md index c08a481f..a8974bad 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ # Morphir Python +Read the [documentation site](https://finos.github.io/morphir-python/) for +[installation](https://finos.github.io/morphir-python/docs/installation/), +[API examples](https://finos.github.io/morphir-python/docs/python-compiler/) and +[CLI workflows](https://finos.github.io/morphir-python/docs/quickstart/). + Python port of [Morphir](https://morphir.finos.org) - a library of tools for building and working with functional domain models. Morphir enables you to write business logic once and use it across multiple platforms, languages, and runtimes. It provides a strongly-typed intermediate representation (IR) that captures the semantics of your domain model. diff --git a/docs/api-reference.md b/docs/api-reference.md new file mode 100644 index 00000000..3c2ec071 --- /dev/null +++ b/docs/api-reference.md @@ -0,0 +1,73 @@ +# Python API reference + +Import the public compiler API from `morphir_tools.python`. It loads the native +compiler lazily when you compile or generate. The private bridge module is not +an application API. See the [complete example](python-compiler.md#use-the-python-api) +and [installation requirements](installation.md) first. + +## Operations + +| Operation | Result | +| --- | --- | +| `compile_sources(sources, *, package, exposed_modules=None, ir_version=IrVersion.V3)` | `CompileResult` | +| `generate_python(distribution)` | `GenerateResult` | +| `native_compiler()` | `NativeCompilerInfo` | + +`sources` is a sequence of `SourceDocument(path, text)`. Paths are relative module +paths with `/` separators. Compilation reads source text statically. `package` +is a Morphir package identity; exposure uses module paths. `IrVersion.V3` is the +default; `IrVersion.V4` is draft. + +## Immutable results and diagnostics + +`CompileResult` is `CompiledV3 | CompiledV4 | CompileFailure`. A success has a +version-specific `distribution` and `warnings`. `GenerateResult` is +`Generated | GenerationFailure`; success contains ordered `GeneratedArtifact` +values with `path` and `text`. `LibraryModel` identifies supported input models. + +`CompilerDiagnostic` has `code`, `severity`, `message`, optional `source` and +`ir_path`. `Severity` distinguishes error, warning, info and hint. A +`SourceLocation` contains the `path`, `start` and `end` `SourcePosition` values. +Positions use zero-based lines and UTF-16 characters. `render()` formats a +human-readable location; use fields and codes in application logic. + +`NativeCompilerUnavailable` reports a missing or incompatible installation. +`NativeCompilerError` reports an internal compiler failure. +`EXPECTED_BRIDGE_API` and `EXPECTED_NATIVE_VERSION` describe the private package +pair expected by this CLI release. They are diagnostic information, not knobs +to override a version mismatch. + +## Checked signatures + +This reference is synchronized from the candidate public exports. The declared +release examples verify that their used signatures work against the released +pair. The reference check fails if a public export disappears from this page +or its signature snapshot changes without updating the source. + + +```text +EXPECTED_BRIDGE_API +EXPECTED_NATIVE_VERSION +CompileFailure(diagnostics: 'tuple[CompilerDiagnostic, ...]') -> None +CompileResult +CompiledV3(distribution: 'v3_dist.Library', warnings: 'tuple[CompilerDiagnostic, ...]' = ()) -> None +CompiledV4(distribution: 'v4_dist.LibraryDistribution', warnings: 'tuple[CompilerDiagnostic, ...]' = ()) -> None +CompilerDiagnostic(code: 'str', severity: 'Severity', message: 'str', source: 'SourceLocation | None' = None, ir_path: 'tuple[str | int, ...]' = ()) -> None +GenerateResult +Generated(artifacts: 'tuple[GeneratedArtifact, ...]', warnings: 'tuple[CompilerDiagnostic, ...]' = ()) -> None +GeneratedArtifact(path: 'str', text: 'str') -> None +GenerationFailure(diagnostics: 'tuple[CompilerDiagnostic, ...]') -> None +IrVersion(*values) +LibraryModel +NativeCompilerError +NativeCompilerInfo(version: 'str', bridge_api: 'int', compiler_version: 'str') -> None +NativeCompilerUnavailable +Severity(*values) +SourceDocument(path: 'str', text: 'str') -> None +SourceLocation(path: 'str', start: 'SourcePosition', end: 'SourcePosition') -> None +SourcePosition(line: 'int', character: 'int') -> None +compile_sources(sources: 'Iterable[SourceDocument]', *, package: 'str', exposed_modules: 'Sequence[str] | None' = None, ir_version: 'IrVersion' = ) -> 'CompileResult' +generate_python(distribution: 'LibraryModel | object') -> 'GenerateResult' +native_compiler() -> 'NativeCompilerInfo' +``` + diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 00000000..dee81ab4 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,79 @@ +# CLI reference + +The Python CLI executable is `morphir-python`, supplied by the `morphir-python` +distribution. It is independent of the two executables called `morphir` in the +[integration guides](compatibility.md). Use [installation](installation.md) +and the [round-trip quickstart](quickstart.md) to begin. + +## Compile Python + +`compile` reads one `.py` file or a directory tree. A directory's relative file +paths establish module identity. Hidden directories, caches and symlinks are +skipped. Compilation defaults to format 3.0.0; choosing v4 prints a draft notice. +Without exposure options, every module is public. + + +```text +morphir-python compile +source [required]: A Python file, or a directory of Python files. +--package [required]: Morphir package name, such as acme/example. +--output, -o [required]: Where to write the IR document. +--ir-version [default: Target.V3]: The IR version to write. +--expose: Module to make public, such as acme/models. Repeat for more. Without it every module is public. +``` + + +## Generate Python + +`generate` reads a supported Library document and creates relative module paths. +The output directory must be new or empty. The CLI validates and stages all +files before publishing them; errors leave existing data intact. + + +```text +morphir-python generate +file [required]: A versioned morphir-ir.json file. +--output-dir [required]: A new or empty directory for the Python modules. +``` + + +## Inspect IR + +Inspection reports the document's format/distribution and summary without +loading the native compiler. It does not imply the model is supported by the +Python compiler or every ecosystem host. + + +```text +morphir-python ir inspect +file [required]: A versioned morphir-ir.json file. +``` + + +## Convert IR + +Conversion is explicit. Legacy formats 1/2 can be upgraded, and supported v3/v4 +models can be migrated. Inspect diagnostics before permitting a documented loss +policy. No implicit conversion runs during generation. + + +```text +morphir-python ir convert +file [required]: A versioned morphir-ir.json file. +--to [required]: The IR version to write. +--output, -o [required]: Where to write the result. +--package-version: Package release version to record when upgrading to v4. +--drop-source-locations, --no-drop-source-locations [default: False]: Drop value source locations when downgrading to v3. +--drop-input-attributes, --no-drop-input-attributes [default: False]: Keep only the declared type of each value-definition input when writing v4, which spells an input as a bare type. +--v4-spelling [default: V4Spelling.COMPACT]: The v4 JSON spelling to write: compact, the canonical form, or expanded. Ignored when writing v3. +``` + + +## Exit status and diagnostics + +Successful operations exit zero. Model, format and output errors exit 1 and +write diagnostics to stderr, including stable codes and available locations. +Argument syntax errors use the CLI parser's nonzero status. Warnings remain +distinct from errors and do not make success fail. A failed compilation +preserves an existing output file. See [compiler diagnostics](python-compiler.md#diagnostics) +for codes and [library guidance](library.md) for migration diagnostics. diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..7ba2403b --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,43 @@ +# Choose a Morphir CLI + +Three independently packaged tools appear in these guides. Confirm the provider +as well as the version before copying an integration command. + +| Tool | Qualified identity | Python compiler route | IR and artifacts | +| --- | --- | --- | --- | +| `morphir-python` | Python distribution 0.1.0, core 0.1.0, native 0.1.1 | In-process typed API and CLI | v3 Library by default; explicit draft v4; versioned JSON | +| `finos/morphir` | Rust product CLI 0.4.0-beta.10 | Python Extism bundle 0.4.2, MEP 0.1 | Explicit v3/v4 JSON; compile writes `morphir-ir.json` | +| MoonBit-based `morphir` | Source pin documented in the MoonBit guide | Current `core-wasm-process` and `native-process` adapters do not run the Python Extism guest | Portable MEP 0.2 draft, IR v4 model envelope; Ion default, explicit JSON tuple | + +## Verify the provider + +Use the `morphir-python` console script from your Python environment. For the +Rust host, use the exact executable path produced by the pinned download in +the [extension guide](python-extension.md), then verify `--version`. For the +MoonBit CLI, use its source/packaged entry point at the declared revision as +described in the [MoonBit guide](moonbit.md). A program named `morphir` somewhere +on `PATH` is insufficient evidence of its provider. + +Installation and invocation are separate operations. A repository can contain +a bundle that an adapter cannot execute. Publisher capability claims also do +not prove that a particular host ran the guest successfully. + +## File exchange is a separate contract + +A shared format number does not imply that a plain IR file is a model envelope. +Each interchange recipe must name producer, consumer, IR release, encoding, +distribution kind and retained metadata. Python migration returns explicit +diagnostics and cannot invent a host envelope or source provenance. + +The [finos/morphir workflow](python-extension.md) qualifies Python compile and +generation through the released bundle. The [MoonBit guide](moonbit.md) records +current adapter/envelope refusals and the separate Extism proposal. It makes no +direct Python-extension or successful portable file-exchange claim today. + +## Requalify a claim + +Run the host-recipe check described in the contributor guide whenever a host, +bundle, recipe or relevant contract changes. The evidence records exact input +hashes and versions. Missing downloads fail qualification. Reuse is permitted +only for the same relevant inputs and dependency identities; a stale green run +for another version does not establish support. diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..2af4a67c --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,63 @@ +# Install Morphir Python + +Use CPython 3.14. Choose a package according to what your application needs. + +| Package | Install for | What it installs | +| --- | --- | --- | +| `morphir` | IR models, codecs, migration and SDK functions | No third-party runtime dependencies | +| `morphir-python` | The CLI and `morphir_tools.python` compiler API | `morphir`, Typer and an exact compatible native compiler | +| `morphir-python-native` | Compiler implementation dependency | Installed automatically by `morphir-python` | + +The released quickstarts use `morphir-python==0.1.0`, `morphir==0.1.0` and +`morphir-python-native==0.1.1`. The core and CLI have independent versions. + +```bash +python3.14 -m venv .venv +. .venv/bin/activate +python -m pip install --only-binary=:all: morphir==0.1.0 morphir-python==0.1.0 +morphir-python --help +``` + +On Windows, activate with `.venv\Scripts\Activate.ps1` in PowerShell. +The `--only-binary` option makes a missing wheel fail explicitly instead of +attempting to build Rust. You do not need a Rust toolchain for these wheels. +See the [CLI quickstart](quickstart.md) next. + +## Supported compiler wheels + +The native compiler uses the CPython 3.14 stable ABI. Qualified wheels cover +Linux x86_64 and aarch64 with glibc 2.28 or later, macOS arm64 on 11 or later, +macOS x86_64 on 10.12 or later, and Windows x86_64. Free-threaded CPython and +other platforms are unqualified. A source installation on another platform +requires Rust 1.98 or later and is distinct from the wheel-only quickstart. + +## Use only the library + +```bash +python -m pip install morphir==0.1.0 +``` + +The [IR and SDK examples](library.md) work without the native compiler. Installing +the library does not install the CLI. The SDK functions are normal Python +functions; the compiler accepts a narrower [source subset](python-compiler.md#supported-python). + +## Released and development examples + +The site tracks the repository's main branch. Examples labeled **Released** run +against the exact qualified packages above. IR v4 remains a draft upstream. +Core 0.1.0 includes the compact canonical v4 writer and the explicit `Spelling` +option. Examples labeled **Development** require wheels built from the source +revision under test; their manifest and receipts identify those prerequisites. + +For a development installation, follow [build from source](python-compiler.md#build-from-source). +Keep the CLI/native pair together. An incompatible private bridge raises +`NativeCompilerUnavailable`; reinstall the CLI and its dependencies rather than +importing or calling the private native module directly. + +## Troubleshoot installation + +If pip reports no matching wheel, check Python version, architecture, libc and +whether the interpreter is free-threaded. If `morphir-python` cannot be found, +activate the environment and check its `bin` or `Scripts` directory. A model error +such as `PY004` is a compiler diagnostic; reinstalling packages will not make +unsupported syntax compile. diff --git a/docs/library.md b/docs/library.md new file mode 100644 index 00000000..fe0d16f7 --- /dev/null +++ b/docs/library.md @@ -0,0 +1,82 @@ +# Work with IR and the SDK + +**Released:** CPython 3.14 and `morphir==0.1.0`. These examples need neither the +CLI nor the native compiler. Install only `morphir` as described in +[installation](installation.md#use-only-the-library). + +## Read, write and migrate IR + +Use `morphir.ir.format_version` to identify a document before choosing a codec. +Use the explicit `v3` or `v4` namespace: `morphir.ir` re-exports draft-v4 types, +which differ from v3 types. + + +```python +"""Read, write and migrate an IR document with the dependency-free library.""" + +import json + +from morphir.ir import v3 +from morphir.ir.format_version import read_document +from morphir.ir.migrate import Ok, v3_to_v4, v4_to_v3 +from morphir.ir.v3 import codec + +text = '{"formatVersion":3,"distribution":["Library",[["acme"]],[],{"modules":[]}]}' +version, _ = read_document(text) +library = codec.decode_versioned_distribution(text) +assert isinstance(library, v3.Library) +written = codec.encode_versioned_distribution(library) +assert codec.decode_versioned_distribution(json.dumps(written)) == library + +upgraded = v3_to_v4(library, package_version="1.0.0") +downgraded = v4_to_v3(upgraded.value) +assert isinstance(downgraded, Ok) and downgraded.value == library +assert any(w.code == "package_version_dropped" for w in downgraded.warnings) +print("IR read/write and explicit migration succeeded") +``` + + +The fixture is an empty Library so every step is visible. The v3 writer emits +the lowest format version that fits. A package release version belongs to v4; +downgrading reports `package_version_dropped`. Migration returns immutable +`Ok` or `Err` values. Check diagnostics rather than assume every v4 construct +has a v3 representation. + +The library reads legacy formats 1 and 2 through dedicated read-only codecs. +Format 3.1 adds Specs distributions. The compiler and Python generator accept +supported Library models, not Specs or Applications. Check the +[package reference](https://github.com/finos/morphir-python/blob/main/packages/morphir/README.md#morphir-ir-versions) +for the full version and loss policy. + +## Use SDK values + +The SDK contains pure functions and immutable values. Functions take arguments +in Morphir/Elm order; `Maybe` is a union of `Just` and `Nothing`. + + +```python +"""Use immutable SDK values with pure Python functions.""" + +from morphir.sdk import maybe + +present: maybe.Maybe[int] = maybe.Just(21) +missing: maybe.Maybe[int] = maybe.Nothing() +assert maybe.map(lambda n: n * 2, present) == maybe.Just(42) +assert maybe.with_default(0, missing) == 0 +assert present == maybe.Just(21) +print("SDK Maybe: 42; missing value: 0") +``` + + +This prints `SDK Maybe: 42; missing value: 0`. Other modules include Basics, +Char, String, List, Dict, Set, Result, Tuple, Decimal, Int and Number. The SDK +is usable from ordinary Python, even where the compiler's source subset does +not yet accept a particular construct. + +## Draft-v4 codec features + +**Released in core 0.1.0:** the v4 writer emits compact canonical spellings +and exposes `morphir.ir.v4.codec.Spelling.EXPANDED` for explicit expanded output. +Core 0.0.2 predates this writer change. Both profiles describe draft v4; choosing +one does not establish compatibility with another tool's envelope or provenance +requirements. See [CLI compatibility](compatibility.md). diff --git a/docs/moonbit.md b/docs/moonbit.md new file mode 100644 index 00000000..eac1d47e --- /dev/null +++ b/docs/moonbit.md @@ -0,0 +1,51 @@ +# Connect with the MoonBit-based CLI + +The [MoonBit Morphir CLI](https://github.com/finos/morphir-moonbit/tree/bd3adc52e6b8f6368fdaee50f76535799732f8d6/apps/morphir) +and the Rust `finos/morphir` CLI share the executable name `morphir`. This guide +describes the MoonBit source at `bd3adc52e6b8f6368fdaee50f76535799732f8d6`. +It does not claim that the Python Extism bundle runs through this version. + +## Current adapter limit + +The Python bundle imports the Extism host ABI and speaks MEP 0.1. The MoonBit +portable `core-wasm-process` adapter expects a core-WASM guest using +`morphir_bytes_v1` callbacks, a binary Ion manifest and MEP 0.2 draft. The +`native-process` adapter is another explicit binding. Neither is an Extism host. +Renaming the Python `.wasm` to `model.wasm` does not supply a compatible ABI. + +The separate [Extism proposal](https://github.com/finos/morphir-moonbit/tree/DamianReeves/propose-extism-python/openspec/changes/qualify-extism-python) +is tracked by `morphir-ukvn.5.98`. It proposes a pinned shared Rust host adapter, +protocol negotiation and qualification with the real Python bundle. The proposal does not establish released support. Until it is independently qualified, use +the [Rust-host recipe](python-extension.md) to invoke the Python extension or +use the [Python API](python-compiler.md) directly. + +## File interchange limits + +The portable MoonBit extension path generates from a MorphirModel envelope, +`model/model-envelope.json`, with a validated payload. Its current tuple uses +envelope `0.1.0-draft.1`, IR `4.0.0`, and either `morphir-model`/`ion-binary` or +`morphir-ir`/`morphir-json`. Compilation can refuse JSON if it would lose source +and naming provenance. Python's versioned `model.json` is a plain IR document. + +The documentation qualification probes a plain Python v3/v4 Library at the +MoonBit portable generation boundary. The expected result is an explicit +envelope refusal before guest execution, with exit status 2 in this pinned CLI. +That is a checked incompatibility, +not a recipe for a successful conversion. No ad hoc envelope, undocumented +provenance drop, or automatic codec fallback is recommended. + +To repeat a probe, use the pinned source's own CLI entry point and provide +the plain `model.json` from the [quickstart](quickstart.md) as the generation +input. Supply a syntactically valid portable pin as required by that CLI so +the check reaches model validation. The qualification runner records the exact +command, input and diagnostic; the upstream +[extension guide](https://github.com/finos/morphir-moonbit/blob/bd3adc52e6b8f6368fdaee50f76535799732f8d6/apps/morphir/docs/extension.md) +defines the full pin schema and invocation contract. + +## Installation is not selection + +The portable invocation requires an explicit extension pin. Local repository +commands and installed selection have separate constraints. Installed native +selection is tracked by `morphir-ukvn.5.64.30`. The site will add a working +Python connection only after the Extism adapter and the exact bundle/host pair +pass their own qualification. diff --git a/docs/python-compiler.md b/docs/python-compiler.md index 2551ac7c..d9e61b29 100644 --- a/docs/python-compiler.md +++ b/docs/python-compiler.md @@ -5,8 +5,11 @@ generates Python from Morphir IR. It does both in process, through a native compiler that installs automatically with the package. You do not need Rust or a separate Morphir executable. +The [released installation guide](installation.md) records the tested package +pair. This page also notes development-only codec behavior where it differs. + The compiler is the Rust crate -[`morphir-python-binding`](../rust/crates/morphir-python-binding/README.md). The +[`morphir-python-binding`](https://github.com/finos/morphir-python/blob/main/rust/crates/morphir-python-binding/README.md). The same crate is the `morphir-python` Morphir extension, so the Python API, the CLI, native Rust callers and Morphir extension hosts give the same results and the same diagnostic codes. @@ -44,7 +47,10 @@ compiler from the source archive, which needs Rust 1.98 or later; see ## Use the Python API + ```python +"""Compile source strings and generate typed artifacts in memory.""" + from morphir_tools.python import ( CompiledV3, CompileFailure, @@ -97,6 +103,7 @@ assert isinstance(generated, Generated) for artifact in generated.artifacts: print(artifact.path) ``` + `compile_sources` takes the source documents of one package, its Morphir package name, and two options: @@ -122,8 +129,9 @@ or `GenerationFailure`. A Specs or Application distribution fails with `unsupported_distribution`. Convert legacy formats 1 and 2 with `morphir-python ir convert` first. -Generation reads both v4 spellings, the compact one the compiler and other -Morphir tools write and the expanded one the Python codec writes. Two limits +Generation reads compact and expanded v4 spellings. Core 0.1.0 writes compact +canonical JSON by default and offers explicit expanded output; see the +[library guide](library.md#draft-v4-codec-features). Two limits apply: - IR v3 integer literals must fit in 64 bits. Compile to v4 for larger integers; @@ -216,7 +224,7 @@ The compiler accepts this subset of Python 3.12 and later, in both IR versions: Anything else is rejected with a diagnostic, never compiled partially. This includes assignments, loops, methods, defaults, containers, optional fields, boolean operators, other imports, and constructor calls inside functions. The -[compiler README](../rust/crates/morphir-python-binding/README.md#current-boundary) +[compiler README](https://github.com/finos/morphir-python/blob/main/rust/crates/morphir-python-binding/README.md#current-boundary) lists the exact boundary. Compilation reads source text only. It never imports or runs the source, and @@ -242,3 +250,25 @@ To build only the native wheel, run `packages/morphir-python-native`. Its source archive contains the whole Rust workspace it needs, so `pip install morphir_python_native-*.tar.gz` works without a checkout when Rust is on `PATH`. + +## Handle a model error + +This example reports `PY004 error rules.py` for unsupported assignment. + + +```python +"""Handle an unsupported model without exceptions or partial output.""" + +from morphir_tools.python import CompileFailure, SourceDocument, compile_sources + +result = compile_sources( + [SourceDocument("rules.py", "answer = 42\n")], package="acme/example" +) +assert isinstance(result, CompileFailure) +diagnostic = result.diagnostics[0] +assert diagnostic.code == "PY004" +assert diagnostic.source is not None +assert diagnostic.source.path == "rules.py" +print(diagnostic.code, diagnostic.severity.value, diagnostic.source.path) +``` + diff --git a/docs/python-extension.md b/docs/python-extension.md index 523323a9..5ce7d5cb 100644 --- a/docs/python-extension.md +++ b/docs/python-extension.md @@ -3,7 +3,7 @@ The `morphir-python` extension lets a Morphir host, such as the Rust `morphir` CLI, compile Python with `morphir compile --language python` and generate Python with `morphir generate --target python`. It is the -[`morphir-python-binding`](../rust/crates/morphir-python-binding/README.md) +[`morphir-python-binding`](https://github.com/finos/morphir-python/blob/main/rust/crates/morphir-python-binding/README.md) compiler built as a WebAssembly guest that speaks the Morphir extension protocol (MEP). It is the same compiler that the `morphir-python` Python package runs natively, so both give the same IR and the same diagnostic codes. @@ -29,36 +29,59 @@ The native API reports compiler version `0.4.2`. Use `0.4.2` or later with hosts that require an explicit multi-document claim; `0.4.1` omits that claim. -The declaration is in [`rust/extension.toml`](../rust/extension.toml). - -## Install a release bundle - -A release contains the WASM guest, its `.sha256` file and a version-2 release -descriptor. Publishing it needs Morphir CLI `0.4.0-beta.7` or later. Rename the -descriptor to `release.json` before you publish it: - -```console -gh release download extension/python/v --repo finos/morphir-python --dir python-bundle -mv python-bundle/morphir-python-binding-.release.json python-bundle/release.json -morphir extension repository init .morphir/python-index -morphir extension repository add python-local --directory .morphir/python-index -morphir extension repository publish python-local --bundle python-bundle -morphir extension repository verify python-local -morphir extension install --repository python-local morphir-python -``` - -Installation verifies the WASM digest and records its capability claims. The CLI -shows WASM claims as `unchecked`: it records the claims the guest declared and -does not probe them. After installation, compilation and generation work -offline. - -## Compile and generate - -Create a project with `src/acme/models.py` and `src/acme/rules.py` (see the -[Python compiler guide](python-compiler.md#use-the-python-api) for supported -Python) and this `morphir.toml`: - -```toml +The declaration is in [`rust/extension.toml`](https://github.com/finos/morphir-python/blob/main/rust/extension.toml). + +## Install the qualified release bundle + +This recipe uses **finos/morphir 0.4.0-beta.10** and Python extension **0.4.2**. +Use the Rust product CLI, not the MoonBit executable of the same name. The +[compatibility guide](compatibility.md) explains the difference. CPython is not +needed to run the installed WASM guest. + +For a reproducible setup from this checkout, `python3 scripts/docs_hosts.py +prepare` downloads the pinned CLI and bundle, verifies the release archive and +both bundle file SHA-256 digests, renames the descriptor to `release.json`, and +prints the absolute `MORPHIR_HOST` and `PYTHON_BUNDLE` paths. Pins live in +[site/integrations.json](https://github.com/finos/morphir-python/blob/main/site/integrations.json) +and [rust/extension.toml](https://github.com/finos/morphir-python/blob/main/rust/extension.toml). +Run the printed `--version` command and check it reports 0.4.0-beta.10. + +For manual acquisition, download the 0.4.2 `.wasm`, `.wasm.sha256` and `.release.json` assets +from the [extension release](https://github.com/finos/morphir-python/releases/tag/extension/python/v0.4.2). +Verify the SHA-256 values in the pin file before using them, then rename the +`.release.json` descriptor to `release.json` in the same bundle directory. +The WASM digest is +`10acafa0f0282c5240e9620a5f2bb21629cc7580b9add528e7b77c7e54db22b5`. +The descriptor digest is +`72770f73f05a72535fd8c86ad4629175019037441a758b827c1ba47cc9a52ebd`. +The host release has platform-specific archives; use the declared target and +hash rather than an unversioned executable from PATH. + +## Install, discover and invoke + +Use a fresh project directory with the `src/acme/` model files from the +[quickstart](quickstart.md), then set `MORPHIR_HOST` and `PYTHON_BUNDLE` to those +verified absolute paths. This POSIX-shell recipe creates an isolated Morphir +home, publishes to a local index, installs the bundle and checks discovery +before compiling. It is the exact recipe run by documentation qualification. + + +```bash +#!/usr/bin/env bash +set -euo pipefail +# MORPHIR_HOST is the verified finos/morphir 0.4.0-beta.10 executable. +# PYTHON_BUNDLE is the verified 0.4.2 bundle directory containing release.json. +: "${MORPHIR_HOST:?Set the verified Rust-host executable path}" +: "${PYTHON_BUNDLE:?Set the verified bundle directory}" +"$MORPHIR_HOST" --version +export MORPHIR_HOME="$PWD/.morphir-home" +"$MORPHIR_HOST" extension repository init .morphir/python-index +"$MORPHIR_HOST" extension repository add python-local --directory .morphir/python-index +"$MORPHIR_HOST" extension repository publish python-local --bundle "$PYTHON_BUNDLE" +"$MORPHIR_HOST" extension repository verify python-local +"$MORPHIR_HOST" extension install --repository python-local morphir-python +"$MORPHIR_HOST" extension list +cat > morphir.toml <<'TOML' [project] name = "acme/example" version = "0.1.0" @@ -67,16 +90,25 @@ source_directory = "src" [frontend] language = "python" emit_parse_stage = false +TOML +"$MORPHIR_HOST" compile --language python --input src --ir-version 3.0.0 --output compiled +"$MORPHIR_HOST" generate --target python --input compiled/morphir-ir.json --output host-generated +"$MORPHIR_HOST" compile --language python --input host-generated --ir-version 3.0.0 --output host-rebuilt ``` - -Then run: - -```console -morphir compile --language python --input src --ir-version 3.0.0 --output compiled -morphir generate --target python --input compiled/morphir-ir.json --output generated -``` - -Omit `--ir-version`, or use `4`, for draft IR v4. + + +The installed extension supplies the `python` frontend and target. Compilation +writes `compiled/morphir-ir.json`; generation writes the two modules beneath +`host-generated/acme/`. Recompilation checks equivalent IR. For this CLI, +omitting `--ir-version`, or selecting `4`, requests draft v4. The Python CLI +instead defaults to v3, so recipes state the version explicitly. + +Installation records capability claims as `unchecked`; the runtime compile and +generate checks provide execution evidence. The existing bundle qualifier also +removes the local index, blocks network access and repeats v3/v4 round trips. +Use `python3 scripts/docs_hosts.py qualify` to repeat the displayed workflow and +offline checks. Receipts record pins, file hashes, host identity, checker and +input digest. Download failure never counts as a passing qualification. ## Build a development bundle diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 00000000..285f3c13 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,72 @@ +# A Python model round trip + +**Released:** CPython 3.14, `morphir-python==0.1.0`, `morphir==0.1.0`, native +compiler `0.1.1`. Follow [installation](installation.md) first. This recipe uses +a POSIX shell such as Bash. The CLI commands themselves also work on Windows; +create the same source files in your editor there. + +## Compile, inspect, generate and recompile + +Start in a new directory. This complete example defines an immutable decision +type and a pure function in another module. It uses local imports and annotated +function arguments. + + +```bash +#!/usr/bin/env bash +set -euo pipefail +mkdir -p src/acme +cat > src/acme/models.py <<'PYTHON' +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved +PYTHON +cat > src/acme/rules.py <<'PYTHON' +from __future__ import annotations +from .models import Decision + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second +PYTHON +morphir-python compile src --package acme/example --output model.json +morphir-python ir inspect model.json +morphir-python generate model.json --output-dir generated +morphir-python compile generated --package acme/example --output rebuilt.json +morphir-python ir convert model.json --to v4 --output model-v4.json +``` + + +`model.json` is a format-3 Library. `ir inspect` reports its format and module +summary. Generation creates `generated/acme/models.py` and +`generated/acme/rules.py`. Recompilation writes `rebuilt.json` with equivalent +supported definitions. Conversion explicitly writes draft v4 to `model-v4.json`. +Conversion diagnostics report information a format cannot retain. Generated +Python needs no Morphir runtime dependency. + +The example keeps semantics rather than comments or source formatting. Running +generation again into the same nonempty directory fails and preserves those +files. Choose a new output directory for another generation. + +## Change the version or module access + +`compile --ir-version v4` requests draft-v4 compiler output directly. With no +`--expose`, every module is public. Repeat `--expose acme/models` to select public +modules. Recompile with the same exposure/package/version options when comparing +models. See [command reference](cli-reference.md) for diagnostics and options. + +To perform the same work without files, use the +[in-memory compiler API](python-compiler.md#use-the-python-api). +The [example sources](https://github.com/finos/morphir-python/tree/main/examples) +are also available as an [archive](/assets/downloads/examples.zip). diff --git a/docs/site-foundation.md b/docs/site-foundation.md new file mode 100644 index 00000000..44cf6d42 --- /dev/null +++ b/docs/site-foundation.md @@ -0,0 +1,40 @@ +# Site foundation and upstream work + +This site consumes [Telescope 0.2.1](https://github.com/moonrockz/telescope/releases/tag/v0.2.1). +It reuses static rendering, navigation, search, themes, page contents and code +copying. Python example execution stays in this repository. Telescope is a +build-time dependency and adds no Python runtime dependencies. + +## Python API reference + +Telescope's current API adapter reads `moon doc` metadata. It does not read +Python type annotations. We maintain concise [Python reference pages](api-reference.md) +and verify their symbols/signatures against installed packages. This is a +non-blocking consumer choice. A Python metadata adapter would be a reusable +extension if a larger generated reference becomes necessary. + +## Tested source snippets + +Telescope 0.2.1 does not include checked source regions. The consumer sync/check +utility maps fenced snippets to canonical examples, while domain tests check +their behavior. Telescope's `standardize-tutorial-components` intent is already +exploring reusable tutorial components. We do not depend on that unmerged work. +Record its durable proposal link when published and avoid duplicate reports. + +## Report a limitation + +Before adding a second renderer or patching a private copy, reproduce a missing +generic feature with the pinned CLI in a minimal consumer. Search +[Telescope issues](https://github.com/moonrockz/telescope/issues) and existing +proposals. Report the version, fixture, commands, expected/actual behavior, +consumer impact and acceptance criteria, and link the report here with any +temporary workaround. A workaround does not turn an unmet acceptance criterion +into a passing check. Adopt upstream fixes through a pinned published release. + +## Repository source links + +Telescope 0.2.1 treats existing files outside discovered roots as missing. +[Telescope issue 10](https://github.com/moonrockz/telescope/issues/10) records the +minimal reproduction and requested source-link contract. Explicit GitHub URLs +are the current non-blocking workaround; internal published-page links remain +relative and checked. diff --git a/docs/site.md b/docs/site.md new file mode 100644 index 00000000..25f7cec9 --- /dev/null +++ b/docs/site.md @@ -0,0 +1,91 @@ +# Contribute to the documentation + +The site uses the published Telescope 0.2.1 native CLI. Its archive URLs and +SHA-256 digests are pinned in [site/toolchain.json](https://github.com/finos/morphir-python/blob/main/site/toolchain.json). +Local preview is qualified on macOS arm64, and CI uses Linux x64. Other site +builder platforms are unqualified. Python 3.14 is sufficient to install and run +the stock builder. No MoonBit compiler or neighboring Telescope checkout is used. + +## Build and preview + +From the repository root, run: + +```bash +python3 scripts/site_tool.py setup +python3 scripts/site_tool.py build +python3 scripts/site_tool.py check +python3 scripts/site_tool.py serve +``` + +Open `http://127.0.0.1:4173/morphir-python/`. Stop preview with Ctrl-C. +The server serves an existing build; rebuild after editing a page. The equivalent +mise commands are `site:setup`, `site:build`, `site:check` and `site:serve`. +The direct Python commands avoid installing the repository's Rust toolchain for +a prose-only edit. + +Use `--base / --out .dev/site-root` on build/check/serve to test root hosting. +The default output is `_site/`, with the Pages base `/morphir-python/`. A build +checks links and resources before replacing prior output. Templates, OpenSpec, +local planning artifacts and generated output stay outside the published tree. + +## Write a guide + +Keep Markdown under `docs/`, add it to the sidebar in `site/site.json`, and use +relative links to sources and other guides. The home page lives in +`site/pages/index.md`. Keep headings descriptive so search and the page contents +list are useful. Use language-tagged fenced code without shell prompt characters. +Existing guide paths remain useful on GitHub. + +Runnable examples have canonical files under `examples/`. Their prerequisite +manifest distinguishes released and development packages. Synchronize marked +snippets with `python3 scripts/docs_examples.py sync`; `check` refuses drift. +Run `mise run docs:test` for installed-example, type-check and snippet checks. +Compilation itself never imports submitted source. Verification of generated +behavior runs in its own process. + +Run the browser suite with `npm ci --prefix site/browser`, +`npm exec --prefix site/browser -- playwright install chromium`, and +`npm test --prefix site/browser`. It checks desktop/phone layouts, themes, +search, navigation, copying and keyboard focus at both deployment bases. +Screenshots and reports are under `.dev/site-browser/`. + +## Qualify host instructions + +`python3 scripts/docs_hosts.py prepare` verifies/downloads the declared host and +bundle and prints their absolute paths. `qualify` runs the exact displayed +Rust-host recipe, the existing offline v3/v4 bundle qualification, and two +MoonBit plain-IR refusal probes. It needs the installed Python compiler, mise, +Node and access to the pinned source/tool releases. It builds the MoonBit CLI +in this repository's disposable `.dev/` tool cache, without changing its repo. + +To change a supported host claim, update `site/integrations.json`, the recipe +and guides together, run `python3 scripts/docs_hosts.py qualify`, then inspect +`.dev/docs-hosts/report.json`. Store a reviewed reusable report at +`site/evidence/hosts.json`. Run `python3 scripts/docs_hosts.py verify --report +site/evidence/hosts.json` to check it still matches every relevant input hash. +Changing any recipe, fixture, pin or checker invalidates reuse. Network or +tool acquisition failures remain failed qualification, with no successful +receipt written. CI produces fresh evidence on relevant changes and full runs. + +## Publish and recover + +Pull requests validate without production credentials. After successful main CI, the publication workflow validates the producer and +promotes the exact checked output with its source, tool and example +evidence manifest, after required checks succeed. A failed, missing or +unexpectedly skipped required result blocks promotion. The deploy job alone +uses Pages/OIDC write permissions and the `github-pages` environment. + +A maintainer must enable GitHub Pages with **GitHub Actions** as the build +source and approve deployment when the repository environment requires it. +Do not change settings or merge merely to test a preview. A manual main CI run repeats qualification before publication. The separate +Publish documentation workflow accepts a successful main CI run ID for restoring +its retained artifact. It refuses pull-request, fork, failed and unrelated +workflow producers, and checks out the producer SHA before verification. Retain the verified Pages artifact, +its digest and the producing run for 30 days. To recover, rerun deployment of +that artifact from its original authorized run without rebuilding, or revert +the content and validate a new main revision. The deployment receipt identifies +the source and artifact selected. A site rollback does not roll back packages. + +For tool updates, change the exact version, URL and digest together, then repeat +clean installs, base-path checks and browser tests. See +[foundation decisions](site-foundation.md) for upstream reports and limitations. diff --git a/examples/cli.sh b/examples/cli.sh new file mode 100644 index 00000000..0f230f44 --- /dev/null +++ b/examples/cli.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +mkdir -p src/acme +cat > src/acme/models.py <<'PYTHON' +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved +PYTHON +cat > src/acme/rules.py <<'PYTHON' +from __future__ import annotations +from .models import Decision + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second +PYTHON +morphir-python compile src --package acme/example --output model.json +morphir-python ir inspect model.json +morphir-python generate model.json --output-dir generated +morphir-python compile generated --package acme/example --output rebuilt.json +morphir-python ir convert model.json --to v4 --output model-v4.json diff --git a/examples/compiler_api.py b/examples/compiler_api.py new file mode 100644 index 00000000..f6b0a13a --- /dev/null +++ b/examples/compiler_api.py @@ -0,0 +1,53 @@ +"""Compile source strings and generate typed artifacts in memory.""" + +from morphir_tools.python import ( + CompiledV3, + CompileFailure, + Generated, + SourceDocument, + compile_sources, + generate_python, +) + +models = SourceDocument( + "acme/models.py", + """\ +from __future__ import annotations +from dataclasses import dataclass + +@dataclass(frozen=True) +class Pending: + pass + +@dataclass(frozen=True) +class Approved: + amount: float + +type Decision = Pending | Approved +""", +) +rules = SourceDocument( + "acme/rules.py", + """\ +from __future__ import annotations +from .models import Decision + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second +""", +) + +compiled = compile_sources([models, rules], package="acme/example") +if isinstance(compiled, CompileFailure): + for diagnostic in compiled.diagnostics: + print(diagnostic.severity.value, diagnostic.render()) + raise SystemExit(1) +assert isinstance(compiled, CompiledV3) + +generated = generate_python(compiled.distribution) +assert isinstance(generated, Generated) +for artifact in generated.artifacts: + print(artifact.path) diff --git a/examples/diagnostics.py b/examples/diagnostics.py new file mode 100644 index 00000000..51bd5841 --- /dev/null +++ b/examples/diagnostics.py @@ -0,0 +1,13 @@ +"""Handle an unsupported model without exceptions or partial output.""" + +from morphir_tools.python import CompileFailure, SourceDocument, compile_sources + +result = compile_sources( + [SourceDocument("rules.py", "answer = 42\n")], package="acme/example" +) +assert isinstance(result, CompileFailure) +diagnostic = result.diagnostics[0] +assert diagnostic.code == "PY004" +assert diagnostic.source is not None +assert diagnostic.source.path == "rules.py" +print(diagnostic.code, diagnostic.severity.value, diagnostic.source.path) diff --git a/examples/extension.sh b/examples/extension.sh new file mode 100644 index 00000000..7057c726 --- /dev/null +++ b/examples/extension.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +# MORPHIR_HOST is the verified finos/morphir 0.4.0-beta.10 executable. +# PYTHON_BUNDLE is the verified 0.4.2 bundle directory containing release.json. +: "${MORPHIR_HOST:?Set the verified Rust-host executable path}" +: "${PYTHON_BUNDLE:?Set the verified bundle directory}" +"$MORPHIR_HOST" --version +export MORPHIR_HOME="$PWD/.morphir-home" +"$MORPHIR_HOST" extension repository init .morphir/python-index +"$MORPHIR_HOST" extension repository add python-local --directory .morphir/python-index +"$MORPHIR_HOST" extension repository publish python-local --bundle "$PYTHON_BUNDLE" +"$MORPHIR_HOST" extension repository verify python-local +"$MORPHIR_HOST" extension install --repository python-local morphir-python +"$MORPHIR_HOST" extension list +cat > morphir.toml <<'TOML' +[project] +name = "acme/example" +version = "0.1.0" +source_directory = "src" + +[frontend] +language = "python" +emit_parse_stage = false +TOML +"$MORPHIR_HOST" compile --language python --input src --ir-version 3.0.0 --output compiled +"$MORPHIR_HOST" generate --target python --input compiled/morphir-ir.json --output host-generated +"$MORPHIR_HOST" compile --language python --input host-generated --ir-version 3.0.0 --output host-rebuilt diff --git a/examples/library.py b/examples/library.py new file mode 100644 index 00000000..899fc9eb --- /dev/null +++ b/examples/library.py @@ -0,0 +1,21 @@ +"""Read, write and migrate an IR document with the dependency-free library.""" + +import json + +from morphir.ir import v3 +from morphir.ir.format_version import read_document +from morphir.ir.migrate import Ok, v3_to_v4, v4_to_v3 +from morphir.ir.v3 import codec + +text = '{"formatVersion":3,"distribution":["Library",[["acme"]],[],{"modules":[]}]}' +version, _ = read_document(text) +library = codec.decode_versioned_distribution(text) +assert isinstance(library, v3.Library) +written = codec.encode_versioned_distribution(library) +assert codec.decode_versioned_distribution(json.dumps(written)) == library + +upgraded = v3_to_v4(library, package_version="1.0.0") +downgraded = v4_to_v3(upgraded.value) +assert isinstance(downgraded, Ok) and downgraded.value == library +assert any(w.code == "package_version_dropped" for w in downgraded.warnings) +print("IR read/write and explicit migration succeeded") diff --git a/examples/manifest.json b/examples/manifest.json new file mode 100644 index 00000000..410edc9e --- /dev/null +++ b/examples/manifest.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "python": "3.14", + "released": { + "morphir": "0.1.0", + "morphir-python": "0.1.0", + "morphir-python-native": "0.1.1" + }, + "examples": [ + { + "source": "examples/compiler_api.py", + "packages": "compiler", + "stdout": "acme/models.py\nacme/rules.py\n" + }, + { + "source": "examples/diagnostics.py", + "packages": "compiler", + "stdout": "PY004 error rules.py\n" + }, + { + "source": "examples/library.py", + "packages": "library", + "stdout": "IR read/write and explicit migration succeeded\n" + }, + { + "source": "examples/sdk.py", + "packages": "library", + "stdout": "SDK Maybe: 42; missing value: 0\n" + } + ] +} diff --git a/examples/model/acme/models.py b/examples/model/acme/models.py new file mode 100644 index 00000000..cc4c9b1e --- /dev/null +++ b/examples/model/acme/models.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Pending: + pass + + +@dataclass(frozen=True) +class Approved: + amount: float + + +type Decision = Pending | Approved diff --git a/examples/model/acme/rules.py b/examples/model/acme/rules.py new file mode 100644 index 00000000..ead6733d --- /dev/null +++ b/examples/model/acme/rules.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from .models import Decision + + +def choose(flag: bool, first: Decision, second: Decision) -> Decision: + if flag: + return first + else: + return second diff --git a/examples/sdk.py b/examples/sdk.py new file mode 100644 index 00000000..81aad8ef --- /dev/null +++ b/examples/sdk.py @@ -0,0 +1,10 @@ +"""Use immutable SDK values with pure Python functions.""" + +from morphir.sdk import maybe + +present: maybe.Maybe[int] = maybe.Just(21) +missing: maybe.Maybe[int] = maybe.Nothing() +assert maybe.map(lambda n: n * 2, present) == maybe.Just(42) +assert maybe.with_default(0, missing) == 0 +assert present == maybe.Just(21) +print("SDK Maybe: 42; missing value: 0") diff --git a/openspec/changes/add-documentation-site/.openspec.yaml b/openspec/changes/add-documentation-site/.openspec.yaml new file mode 100644 index 00000000..dda53b8a --- /dev/null +++ b/openspec/changes/add-documentation-site/.openspec.yaml @@ -0,0 +1,2 @@ +schema: html-design +created: 2026-10-27 diff --git a/openspec/changes/add-documentation-site/design.html b/openspec/changes/add-documentation-site/design.html new file mode 100644 index 00000000..bcbd06aa --- /dev/null +++ b/openspec/changes/add-documentation-site/design.html @@ -0,0 +1,172 @@ + + + + + +add-documentation-site + + + + + + + + +
+

add-documentation-site

+

OpenSpec change · Bead morphir-ukvn.9 · 2026-10-28 · Implemented, awaiting review

+ + + +
+

Context and evidence

+

See the proposal for motivation. The compiler and extension already exist. This design adds a public guide around their present contracts, with executable evidence for the instructions readers follow.

+ + + + + + + + + + +
Observed sourceConsequence
Python compiler guide and its testsReuse the typed API example and fresh-interpreter/type-checking coverage. The current test assumes one page and fixed stdout; expand it for named examples and assertions.
Extension guide and bundle pinsThe Python artifact is an Extism guest. Existing host qualification and bundle metadata are the starting evidence for finos/morphir.
MoonBit site authoring guideFollow its user journeys, tested examples and checked links. Use Telescope rather than copy its custom MoonBit renderer.
MoonBit extension adaptersCore-WASM process and native-process support do not qualify an Extism guest. The portable path uses an IR v4 model envelope and has explicit codec/provenance constraints.
Telescope CLI and 0.2.1 release evidenceA stock native CLI builds Markdown/HTML, checks links, and embeds CSS and browser controls. Its API adapter consumes MoonBit metadata, not Python signatures.
CI selector and CI workflowMost docs currently select no work. Unknown paths, workflow changes and full runs select all categories. Add explicit docs ownership without weakening aggregate failure handling.
+
+ +
+

Goals and boundaries

+

Readers should install the right package, complete a small model round trip, use the IR/SDK library directly, and understand which host can run the extension. A contributor should preview the same site that CI checks and publishes. The contracts are in docs/site, docs/examples and docs/cli-integration.

+

The initial site uses one documentation tree with explicit release and development labels. It does not introduce an evaluator, a browser Python runtime, a benchmark claim about Rust performance, or a new extension adapter. Full historical version trees can follow when releases need them. The user selected current-support documentation; the MoonBit Extism proposal is separate and does not gate this site.

+
+ +
+

Decisions

+

SITE-1. Consume the published Telescope CLI

+

Start with Telescope 0.2.1. Pin the release archive and expected SHA-256 in repository-owned setup metadata for Linux x64 CI and macOS arm64 local use. Verify the binary version after installation. These targets have published qualification; document other platforms as unqualified. Telescope is build tooling, outside the Python runtime dependency graph. A stock binary requires no MoonBit compiler or neighboring checkout.

+

Use site/site.json, site/pages/ for the landing page, existing docs/ for guides and site/assets/ for repository assets. Explicit discovery/exclusions must keep templates, planning documents, OpenSpec changes and generated output out of navigation and search. Proposed mise tasks are site:setup, site:build, site:check, site:serve and docs:test. Setup should install only the tools each command needs.

+
{
+  "schema_version": 1,
+  "title": "Morphir Python",
+  "description": "Python models, Morphir IR and language tooling",
+  "base_path": "/morphir-python/",
+  "site_url": "https://finos.github.io/morphir-python/",
+  "repo_url": "https://github.com/finos/morphir-python",
+  "repo_ref": "main",
+  "packages": [],
+  "source_roots": ["docs", "site/pages"]
+}
+
+

This is a configuration starting point, not the final navigation manifest. Configure exclusions, navigation and static resources using the pinned version's supported contract. Build to _site/; test both / and /morphir-python/. Keep existing guide URLs in the repository useful and add site links to READMEs. Copying MoonBit's renderer would duplicate work now owned by Telescope; a separate Python documentation framework would split that shared foundation.

+ +

DOCS-1. Organize around complete user journeys

+ + + + + + + + + +
NavigationInitial content
StartPackage responsibilities, supported Python/wheel platforms, installation, first CLI round trip and first in-memory API example.
Python APICompiler inputs, typed success/failure results, diagnostics, exposure and generated artifacts; IR codecs, version reading/migration and representative SDK functions.
CLIcompile, generate, ir inspect, ir convert, options, exit behavior, output safety and supported Python subset.
Connect to MorphirHost identification, compatibility table, finos/morphir extension setup, MoonBit limitations and qualified IR exchange or refusal examples.
ContributeBuild/preview/check commands, example authoring, pin updates, host qualification, publication and Telescope upstream reporting.
+

Use one checked multi-module domain example for API, CLI and extension guides, plus small independent IR/SDK examples. Explain the result before reference details. Keep a concise public symbol/command reference checked against exports, signatures and help output. Telescope's MoonBit API importer is unsuitable for Python; explicit Python reference pages avoid adding an unrelated generator to the first delivery.

+ +

DOCS-2. Treat runnable snippets as maintained examples

+

Store canonical Python sources, CLI argument arrays, fixtures and expected results under examples/. Give runnable snippets stable example identifiers and source-region references. A small Python docs utility synchronizes marked fenced regions in committed Markdown and offers a check-only mode that fails on drift. This works with the stock Telescope CLI. Retain source-file links and downloadable examples. Do not copy a tutorial into multiple untracked code blocks.

+

Extend the existing docs test coverage to execute named API examples in isolated interpreters and run mypy/pyright, checking typed results, artifacts and diagnostics. Run CLI recipes in temporary directories against installed distributions, assert semantic round trips, and cover invalid source and nonempty output directories. Test snippets without executing submitted compiler input as part of compilation. Generated model behavior can be checked in a separate verification process, as existing compiler tests do.

+

Record example prerequisites in a small machine-readable manifest. Released quickstarts use an exact qualified package set installed outside the checkout; development examples use the candidate wheels already produced by CI and visibly say they require a source build. The site's main-branch freshness must not imply that an unreleased compact-v4 writer or migration is available in a released wheel. A single tree can carry both labels without a version-assembly pipeline.

+ +

HOST-1. Qualify each CLI independently

+ + + + + + + +
ConsumerCurrent evidenceGuide treatment
morphir-pythonTyped Python API and installed CLI call the native compiler in process. Default compiler output is IR v3; draft v4 is explicit.Release-qualified install and API/CLI round trips, independent of another Morphir executable.
finos/morphirExisting Extism bundle host qualification; descriptor/CLI versions are pinned in rust/extension.toml.Pin an actual released host/bundle pair, download and verify, create/select repository, install, discover capabilities and compile/generate. State IR selection explicitly.
MoonBit-based Morphir CLICore-WASM callback adapter and native-process adapter; its current guide requires a separately qualified Extism adapter.Label direct Python-bundle invocation unsupported. Probe narrow file exchange with the pinned CLI and show either a verified route or an actionable refusal.
+

Give each recipe an executable path and version/provider check so two programs called morphir cannot silently substitute for one another. Record exact host revisions/releases, bundle digest, MEP version, IR version/encoding, model distribution and envelope shape. A plain IR JSON document and a MoonBit model-envelope.json are distinct artifacts. Ion support or a shared IR version alone does not prove a working file exchange. Do not add ad hoc envelope or provenance conversions to make a documentation test pass.

+

Reuse scripts/extension_bundle.py qualification and its existing host rather than invent another bundle packager. Add a bounded host recipe runner for the documented setup. Host-integration evidence contains input/command hashes, dependencies, outputs, diagnostic expectations and the qualification runner identity. Successful support claims require matching evidence. If no MoonBit exchange direction works today, publish the checked incompatibility and follow-up instead of expanding runtime scope.

+

A Codex workspace was dispatched at /Users/damian/orca/workspaces/morphir-moonbit/propose-extism-python, branch DamianReeves/propose-extism-python, to propose Extism support separately. Replace this dispatch reference with its durable proposal/bead link when available. Existing MoonBit installed-selection follow-up morphir-ukvn.5.64.30 is another reason to distinguish installation from invocation.

+ +

UP-1. Report generic foundation gaps to Telescope

+

Use stock rendering and browser controls first. Python compilation, CLI execution and result assertions belong to this consumer. Telescope's current API adapter accepts MoonBit metadata, so use authored, checked Python reference pages. If a reusable Python metadata adapter becomes necessary, report it upstream with a minimal fixture and acceptance criteria before building a private parallel system.

+

The local Telescope checkout already contains an unmerged standardize-tutorial-components intent covering source-region examples and verification metadata. Record that relationship without treating it as a released dependency or modifying its draft. If its proposal later has a durable upstream link, use it rather than file a duplicate. For any blocking limitation, record version, minimal reproduction, expected behavior, consumer impact, workaround and upstream issue/proposal in docs/site-foundation.md. Adopt a fix only through a published, pinned release after checks. No confirmed rendering blocker currently requires a new Telescope issue.

+
+ +
+

Checks and publication

+

CI-1. Select docs work and promote one checked artifact

+

Add docs ownership to the existing CI selector. Keep specific site/example/workflow rules ahead of broad docs, scripts and workflow rules. Published prose/configuration selects static build/check/browser coverage. Example or referenced API/CLI changes also select the relevant installed-package tests, using compatible existing wheel artifacts. Host recipe/pin changes select host qualification. Unknown paths and full runs remain conservative, and selected jobs that fail, cancel, disappear or skip unexpectedly must fail the aggregate.

+ + + + + + + + +
Check and ownerRisk, trigger and deadlineEvidence and failure action
Static site, docs maintainersBroken links, excluded content leaks, deployment paths and unreadable UI. All published prose/config/assets; before merge and deployment.Build/check at both bases; browser smoke at desktop and 360px, both themes, search/menu/copy/focus. Failure blocks the aggregate and publication.
Examples, Python maintainersIncorrect API/CLI advice. Example sources, snippets, dependencies and referenced APIs/commands; before merge and deployment.Snippet drift, type checks, isolated execution and output assertions for the declared package set. Reuse wheel producers when candidate checks are selected.
Host recipes, extension maintainersFalse compatibility claims. Recipe/pin/host-contract changes; before new or revised claims merge or publish.Exact recipe and artifact evidence. Missing evidence blocks the changed claim. Unchanged recipes may reuse evidence only with matching relevant-input hashes and qualified dependencies.
Pages, docs maintainersPublishing unverified bytes. Authorized main push or manual main run; after all required evidence succeeds.Deploy the checked build artifact, with source SHA, Telescope version, package pins and evidence manifest. No rebuild during deployment.
+

Factor reusable site build/check work so the main CI and deployment caller use one definition. Keep deployment in a separate trusted-main publication workflow, with pages: write and id-token: write only on the deploy job and the github-pages environment. PR validation has no deployment credentials. The CI workflow is also reused by package releases, whose callers grant no Pages permission. A separate workflow_run publisher therefore verifies the producer repository, event, main branch, workflow path and successful conclusion before downloading an artifact, then checks out its exact SHA. PR/fork/failed/unrelated producers are refused. A manual run ID provides rollback through the same checks. This avoids requesting publication permissions through package-release callers. Re-run required examples for the main candidate or verify the documented relevant-input equivalence contract; a cache hit alone is insufficient.

+

Initial checks run on Linux x64; validate local installation on macOS arm64. Broader package-platform coverage stays with existing package qualification. No new nightly schedule is needed. Measure representative run duration and artifact reuse during implementation; this proposal promises no unmeasured speedup.

+
+Build-master reassessment and validation evidence +

Implementation af9c725 passed every check in CI run 40637061027, against base f9c9af6. The retained site binds the integrated candidate 302892bc85fdbac838c9b0b850f03a7e3ed48023, Telescope 0.2.1, all three installed-example groups, Linux host qualification and hashes of 20 published files. A local download independently passed artifact verification and Telescope checking. PR 51 has no automated review findings.

+

After rebasing onto the corrected companion release 1f547d1, local full checks passed 2,518 tests, 34 BDD scenarios and 98.35% coverage. The current qualified set is core/CLI 0.1.0, native 0.1.1 and extension 0.4.2. Clean published-wheel, candidate-wheel and library-only examples passed after publication; the strict multi-document check remains enforced. The first refreshed CI attempt reached PyPI before its CLI index entry was visible, so fresh-index acquisition and final CI repeat that qualification before merging. Twelve browser cases passed at both deployment bases. Linux CI measured installed examples at 43 seconds, host qualification at 76 seconds, and site/browser validation at 45 seconds. These are one-run job durations, not queue time, billing or a measured before/after speedup. The compatible native wheel was produced once and reused by tests, bundle qualification, installed examples and packaging.

+

The implementation preserves required failure/cancellation/skip handling, binds evidence and artifact bytes, and refuses PR/fork/failed/unrelated producers. No required coverage was deferred to a schedule. GitHub Pages now reports Actions deployment enabled. A first authorized production promotion and a real production rollback remain workflow follow-up after review and merge.

+ + + + + + + + + + +
CriterionWeightImplemented ratingEvidence and confidence
Risk and coverage fit254Selector/aggregate regressions, broken-build preservation, stale receipt and artifact/producer refusal tests; full CI passed. High confidence for tested contracts.
Feedback and placement203Measured category jobs run independently after their real producer. No prior site baseline; one Linux run gives medium confidence.
Evidence and trust20UCandidate binding and producer refusals pass; first production workflow invocation awaits authorized main publication.
Execution efficiency153Native artifact reuse verified. Small pure wheels are built for early docs feedback before the broader package gate; no compute-saving estimate is claimed.
Ownership and structure104One reusable documentation workflow with explicit example/host inputs, stable aggregate and separate privileged publisher. High confidence from source and CI.
Operability and recovery102Failure diagnostics, 30-day retention and run-ID rollback are documented and their refusal checks pass. A real Pages rollback is untested.
+

The total remains withheld because the 20-weight production trust criterion is unknown. Retain the scoped design provisionally: the all-checks-on-every-prose-edit alternative adds unrelated compiler work, while this selector preserves conservative unknown/full-run behavior and qualifies relevant examples and host changes. The maintainer owns the first production promotion and rollback evidence; a green proposal or PR does not grant merge or deployment approval.

+
+
+ +
+

Risks and trade-offs

+ + + + + + + + + + +
RiskMitigation
MoonBit interoperability cannot be made successful with documentation alone.Checked refusals satisfy accurate current-support guidance. Extism implementation remains a separate upstream proposal.
Telescope 0.2.1 lacks a required reusable feature.Report or link upstream, keep a bounded documented consumer workaround and expose any unmet acceptance criterion.
Main-branch API docs get ahead of released packages.Exact prerequisite manifests, visible release/development labels and installed-artifact checks.
Generated snippets add maintenance cost.One small sync/check utility with explicit markers; consumer tests own domain behavior. Replace with an upstream facility only after a published compatible release.
Host downloads are unavailable or pins change.Retain matching evidence for immutable qualified inputs. Changes to those inputs require fresh qualification; download failure never counts as success.
Pages settings are not enabled.Prepare and verify the complete site artifact and workflow first, then request the specific maintainer setting if necessary.
+
+ +
+

Migration and rollback

+

Deliver the implementation in one PR if practical, with task groups carrying their own tests and docs. Preserve existing guide paths and enhance their content rather than delete the only instructions before the site is live. Add the site skeleton, checked examples and host guidance, then integrate the selector and publication job. Run the pinned consumer installation, project-base checks and browser smoke on the complete candidate.

+

After review and authorized merge, deploy the verified main artifact. If repository settings block it, provide the checked artifact and exact setting request. Roll back a bad publication by redeploying a retained previously verified site artifact or reverting the content change and validating that revision. Keep package/extension releases independent of a site rollback. Archive this OpenSpec change and close task beads only after the agreed implementation evidence exists.

+
+
+ + diff --git a/openspec/changes/add-documentation-site/proposal.md b/openspec/changes/add-documentation-site/proposal.md new file mode 100644 index 00000000..ea477956 --- /dev/null +++ b/openspec/changes/add-documentation-site/proposal.md @@ -0,0 +1,34 @@ +# Proposal + +## Why + +Python users need a discoverable guide to the library, compiler API, CLI and extension. The current repository guides do not provide a public site or explain the different support and IR conventions of the two Morphir CLIs in one place. + +## What Changes + +- Publish a repository-owned documentation site using a pinned release of moonrockz/telescope, with navigation, search, readable code examples and local preview instructions informed by morphir-moonbit's site. +- Add tested examples for Python API compilation/generation, IR codecs and migration, SDK usage, and CLI compile/generate/inspect/convert workflows. Explain package installation, supported platforms, diagnostics and language limits. +- Document the Python extension's qualified installation and invocation through finos/morphir. Publish a compatibility matrix and verified file-based IR interchange recipes for the MoonBit-based CLI, with explicit unsupported cases. +- Keep MoonBit Extism adapter implementation in a separate upstream proposal. This change documents current support and links the follow-up; it does not claim that the existing Python WASM bundle works with the MoonBit core-WASM adapter. +- Add scoped example/site checks and GitHub Pages delivery. Report concrete reusable site limitations to Telescope, linking existing upstream work where it already covers the need. + +## Capabilities + +### New Capabilities + +- `docs/site`: Discoverable, accessible repository documentation, reproducible preview and checked publication. +- `docs/examples`: Executable Python API and CLI learning workflows with explicit package/version prerequisites. +- `docs/cli-integration`: Qualified extension and IR interchange guidance for each Morphir CLI, including compatibility limits. + +### Modified Capabilities + +None. Compiler, extension protocol and IR behavior remain governed by their existing specs. + +## Beads + +- **Tracked by**: `morphir-ukvn.9`. +- **Related**: `morphir-ukvn.1.6` supplies the migrated compiler, native package and extension. The separate MoonBit Extism proposal was dispatched to the Codex workspace `propose-extism-python` in finos/morphir-moonbit; its eventual artifact/bead will be linked without blocking this site's accurate compatibility guidance. + +## Impact + +Documentation under `docs/`, package READMEs, canonical examples and their tests; new `site/` configuration/assets; mise tasks; CI selection and Pages workflows. Telescope is a build-time tool only. Python package runtime dependencies and public APIs do not change. GitHub Pages repository settings require a maintainer action if Actions deployment is not enabled. This proposal authorizes no deployment, runtime adapter changes or upstream Telescope implementation. diff --git a/openspec/changes/add-documentation-site/specs/docs/cli-integration/spec.md b/openspec/changes/add-documentation-site/specs/docs/cli-integration/spec.md new file mode 100644 index 00000000..f54d029f --- /dev/null +++ b/openspec/changes/add-documentation-site/specs/docs/cli-integration/spec.md @@ -0,0 +1,50 @@ +# CLI integration guidance + +## Purpose + +Give users accurate, reproducible instructions for using the Python extension and exchanging IR across the independently implemented Morphir CLIs. + +## ADDED Requirements + +### Requirement: Compatibility guidance identifies the host +The documentation SHALL distinguish morphir-python, finos/morphir and the MoonBit-based Morphir CLI by provider and version. Its compatibility matrix MUST identify extension transport, protocol, IR formats, artifact shape, supported operations and qualification evidence for each host. (bead: morphir-ukvn.9.3) + +#### Scenario: Two CLIs use the same executable name +- **WHEN** a user selects an extension setup recipe +- **THEN** the guide identifies which Morphir implementation must be installed and how to verify it before invoking commands + +### Requirement: Qualified extension setup is reproducible +The finos/morphir guide SHALL document a qualified Python bundle/host pair, checksum verification, repository publication or selection, installation, discovery and compile/generate invocation. It MUST distinguish installing a bundle from selecting it for execution. (bead: morphir-ukvn.9.3) + +#### Scenario: Use a clean extension repository +- **WHEN** a user follows the guide with the pinned bundle and host in a fresh workspace +- **THEN** the host verifies and installs the expected extension, compiles the supplied Python model and generates Python with the documented artifacts + +### Requirement: Unsupported adapters are explicit +The documentation MUST state that the current Python Extism guest is not interchangeable with the MoonBit core-WASM process adapter. Until an Extism adapter is independently qualified, the MoonBit guide SHALL label direct invocation unsupported and link its separate upstream proposal. (bead: morphir-ukvn.9.3) + +#### Scenario: Attempt direct MoonBit invocation +- **WHEN** a user reads how to connect the current Python WASM bundle to the MoonBit CLI +- **THEN** the guide explains the adapter and protocol gap and does not present an unqualified direct-invocation command as working + +### Requirement: IR interchange claims are qualified +Each documented interchange direction SHALL identify the exact producer/consumer pair, model subset, version, encoding and artifact shape. Successful recipes MUST verify the consumer's result; refusal cases MUST identify the incompatibility. The guide MUST distinguish file exchange from invoking the Python extension through that host. (bead: morphir-ukvn.9.3) + +#### Scenario: Exchange a model using a supported route +- **WHEN** a user follows a recipe classified as supported for the qualified versions +- **THEN** the supplied fixture reaches the documented consumer operation through the stated conversion steps with checked semantic results + +#### Scenario: Formats or envelopes differ +- **WHEN** the current host pair cannot exchange the supplied model without unsupported conversion or provenance loss +- **THEN** the guide records a tested refusal and the missing capability rather than suggesting that renaming a plain IR file supplies the required envelope + +### Requirement: Qualification failures cannot silently become support claims +Host integration checks SHALL record pinned host and bundle identities, commands, expected artifacts and diagnostics. Changes to a supported recipe MUST rerun its required qualification before publication. External-host availability problems MUST remain visible and cannot count as a passing compatibility check. (bead: morphir-ukvn.9.3) + +#### Scenario: A supported recipe changes +- **WHEN** an edit changes a host pin, bundle pin or executable setup command +- **THEN** publication requires evidence for that exact combination and changed recipe + +#### Scenario: The host cannot be obtained +- **WHEN** qualification cannot obtain a required host or bundle +- **THEN** the result reports the unavailable prerequisite and blocks a new or revised supported claim diff --git a/openspec/changes/add-documentation-site/specs/docs/examples/spec.md b/openspec/changes/add-documentation-site/specs/docs/examples/spec.md new file mode 100644 index 00000000..d2471e58 --- /dev/null +++ b/openspec/changes/add-documentation-site/specs/docs/examples/spec.md @@ -0,0 +1,58 @@ +# Verified learning examples + +## Purpose + +Help Python users learn the library and tools through reproducible examples whose displayed source and documented results stay consistent with the qualified packages. + +## ADDED Requirements + +### Requirement: Installation guidance identifies package responsibilities +The guides SHALL distinguish the library, CLI distribution and automatically installed native compiler package. Each workflow MUST identify required interpreter/platform support and qualified package versions, separating released capabilities from source-only or draft capabilities. (bead: morphir-ukvn.9.2) + +#### Scenario: Start with an installed package +- **WHEN** a user follows the compiler quickstart on a qualified wheel platform +- **THEN** the guide installs the CLI distribution and its compatible compiler without requiring the user to install Rust + +#### Scenario: Use only the library +- **WHEN** a user follows an IR or SDK example +- **THEN** the guide identifies the library-only installation and does not require the compiler package for that example + +### Requirement: API examples cover both compiler directions +The guides SHALL provide complete examples for in-memory Python compilation, generation from typed IR, and structured diagnostics. Examples MUST explain supported syntax, package/module identity, exposure and IR version selection. (bead: morphir-ukvn.9.2) + +#### Scenario: Compile and generate through Python +- **WHEN** a user runs the API example with the documented prerequisites +- **THEN** the example compiles a multi-module model, inspects its typed result and generates source artifacts with the documented relative paths + +#### Scenario: Understand unsupported source +- **WHEN** a user runs the diagnostic example +- **THEN** the example returns the documented error code and available source location and explains the relevant language limit + +### Requirement: Library examples explain IR and SDK usage +The guides SHALL provide executable examples for reading and writing versioned IR, explicit supported migration and representative SDK operations. They MUST explain format support and refusal or loss diagnostics without implying that every distribution or construct can be compiled or generated. (bead: morphir-ukvn.9.2) + +#### Scenario: Inspect and migrate a model +- **WHEN** a user follows the library example using the supplied fixture +- **THEN** the example reads its version, decodes and writes it, and demonstrates an explicit supported conversion with its documented diagnostics + +### Requirement: CLI examples cover a complete workflow +The guides SHALL include runnable compile, generate, inspect and convert examples, with input fixtures, expected files and exit behavior. They MUST explain output-directory constraints and distinguish semantic round trips from source-text preservation. (bead: morphir-ukvn.9.2) + +#### Scenario: Follow the CLI quickstart +- **WHEN** a user runs the documented compile, inspect, generate and recompile sequence in a fresh directory +- **THEN** the commands produce the documented artifacts and preserve the supported model's normalized semantics + +#### Scenario: Diagnose a command failure +- **WHEN** a documented failing command uses unsupported source or a nonempty generation directory +- **THEN** the observed exit status and diagnostic match the guide and existing user files remain intact + +### Requirement: Displayed examples match checked sources +Every example presented as runnable SHALL have a canonical source and automated verification of its claimed result. Displayed code MUST match that source. API examples SHALL pass the project's type checks; CLI examples SHALL verify commands and outputs in isolated temporary directories. Unsupported platform or external-host cases MUST be labeled with their qualification limits. (bead: morphir-ukvn.9.2) + +#### Scenario: A snippet drifts +- **WHEN** an edit changes a displayed runnable snippet without updating its checked source +- **THEN** documentation verification fails before publication + +#### Scenario: An API or CLI option changes +- **WHEN** a public signature or command option used in an example changes incompatibly +- **THEN** verification detects the obsolete example or reference rather than publishing a successful quickstart claim diff --git a/openspec/changes/add-documentation-site/specs/docs/site/spec.md b/openspec/changes/add-documentation-site/specs/docs/site/spec.md new file mode 100644 index 00000000..4e610e08 --- /dev/null +++ b/openspec/changes/add-documentation-site/specs/docs/site/spec.md @@ -0,0 +1,54 @@ +# Documentation site + +## Purpose + +Provide a public, repository-owned documentation site that users can navigate and contributors can preview and validate before publication. + +## ADDED Requirements + +### Requirement: Documentation is discoverable +The site SHALL expose installation, Python API, CLI, extension integration and contributor guides through navigation and search. Repository and package READMEs SHALL link to the site while retaining useful installation summaries. (bead: morphir-ukvn.9.1) + +#### Scenario: Find a workflow +- **WHEN** a user opens the home page or searches for Python compilation +- **THEN** the user can reach the corresponding runnable API and CLI guides and their prerequisites + +### Requirement: Documentation remains readable across devices +The site SHALL support keyboard navigation, visible focus, light and dark themes, and narrow screens. Code examples SHALL preserve whitespace, identify their language and allow copying without decorative prompt characters. (bead: morphir-ukvn.9.1) + +#### Scenario: Read a guide on a narrow screen +- **WHEN** a reader opens a guide at a 360-pixel viewport in either theme +- **THEN** navigation and content remain usable without page-level horizontal overflow, with long code scrollable within its block + +#### Scenario: Navigate using a keyboard +- **WHEN** a reader uses keyboard controls to open navigation, search and copy an example +- **THEN** each control is reachable and exposes visible focus and an accessible name + +### Requirement: Contributors can reproduce a checked preview +The repository SHALL document a pinned site-tool installation and commands to build, check and serve a local preview. Builds SHALL exclude planning artifacts and validate local pages, resources, fragments and deployment-base paths before publication. (bead: morphir-ukvn.9.1) + +#### Scenario: Preview a project-path deployment +- **WHEN** a contributor follows the guide in a clean checkout and builds for the repository's Pages path +- **THEN** the site passes checks and local preview resolves navigation, search and assets beneath that path + +#### Scenario: Broken content blocks publication +- **WHEN** a page links to a nonexistent local fragment or resource +- **THEN** validation fails with an actionable location and the invalid output is not published + +### Requirement: Publication uses verified output +Site publication SHALL consume the exact output that passed required site and example checks for the selected source revision. Pull requests SHALL receive validation without production publication. Failed, missing or cancelled required checks MUST block publication. (bead: morphir-ukvn.9.4) + +#### Scenario: Publish a checked revision +- **WHEN** an authorized main-branch publication passes its required checks +- **THEN** deployment publishes their verified artifact with recorded source and tool versions + +#### Scenario: Example verification fails +- **WHEN** required example verification fails or is unexpectedly skipped +- **THEN** no production site deployment occurs for that revision + +### Requirement: Reusable site limitations are reported upstream +When a site-foundation limitation obstructs a required experience, contributors MUST record a reproducible report in Telescope's tracker or link existing upstream work, documenting the affected version, consumer need and any temporary workaround. (bead: morphir-ukvn.9.1) + +#### Scenario: A required presentation feature is unavailable +- **WHEN** a contributor confirms the pinned foundation cannot render a required example interaction +- **THEN** the change records an upstream report or existing proposal and labels any workaround and remaining acceptance gap diff --git a/openspec/changes/add-documentation-site/tasks.md b/openspec/changes/add-documentation-site/tasks.md new file mode 100644 index 00000000..baf9375b --- /dev/null +++ b/openspec/changes/add-documentation-site/tasks.md @@ -0,0 +1,42 @@ +# Tasks + +Implement the contracts in `specs/docs/` using the decisions in `design.html`. Scope is current-support documentation; MoonBit Extism implementation remains a separate proposal. Prefer one implementation PR if practical. + +## 1. Establish Telescope site and local preview (morphir-ukvn.9.1) + +- [x] 1.1 Add pinned Telescope 0.2.1 archive/version/checksum setup metadata and `site:setup`; verify clean Linux x64 and macOS arm64 installation without a Telescope checkout or MoonBit compiler and document other platform limits. +- [x] 1.2 Add `site/site.json`, landing page, navigation and static assets with `/morphir-python/` deployment configuration; verify installation, API, CLI, integration and contributor sections are reachable and templates/planning/generated files are absent from navigation and search. +- [x] 1.3 Add `site:build`, `site:check` and `site:serve` tasks and a contributor guide; verify the documented sequence builds/checks/serves at `/` and `/morphir-python/`, preserves prior output on validation failure and reports a deliberately broken fragment. +- [x] 1.4 Add browser smoke coverage for desktop and 360px, light/dark themes, keyboard focus, navigation, search and code copying; verify no page-level overflow and retain screenshots/results for the candidate. +- [x] 1.5 Record the foundation/version/limitations in `docs/site-foundation.md`; link Telescope's existing tutorial intent when durable, and report any confirmed reusable blocker upstream with reproduction and acceptance criteria. Verify each remaining gap has a linked report or an explicit non-blocking consumer workaround. + +## 2. Publish verified Python API and CLI examples (morphir-ukvn.9.2) + +- [x] 2.1 Add canonical multi-module model, compiler API and CLI recipe sources under `examples/`, with a prerequisite/result manifest; verify compile/generate/recompile semantic assertions and diagnostic/output-safety cases against installed packages outside the checkout. +- [x] 2.2 Add focused IR codec/version/migration and SDK examples; verify library-only execution without the native compiler dependency and label draft/source-only behavior according to tested package versions. +- [x] 2.3 Add source-region snippet synchronization and check-only mode; evolve `tests/tools/test_python_docs.py` and add `docs:test` for named examples. Verify stale snippets fail, API examples pass mypy/pyright and fresh-interpreter tests, and CLI recipes run in temporary directories with checked artifacts/status/diagnostics. +- [x] 2.4 Expand the compiler guide and add installation, library/API, command reference and language-limit pages; verify exported symbol/signature and CLI-help consistency plus every runnable block's canonical-source mapping. +- [x] 2.5 Add release/development prerequisite labels and exact package pairing, wheel-platform and troubleshooting guidance; verify released quickstarts in a clean wheel-only installation without Rust and development examples against the compatible candidate wheels. +- [x] 2.6 Update repository and package READMEs to link site journeys while retaining useful installation summaries; verify relative links in GitHub sources and rendered site checks. + +## 3. Document qualified Morphir CLI integrations (morphir-ukvn.9.3) + +- [x] 3.1 Add a versioned compatibility matrix and explicit executable/provider checks for morphir-python, finos/morphir and the MoonBit CLI; verify transport, MEP, IR, encoding, envelope and supported-operation claims against pinned source/release evidence. +- [x] 3.2 Replace placeholder-only extension setup with a concrete qualified finos/morphir host/bundle recipe covering checksum verification, repository selection/publication, installation, discovery and compile/generate; reuse existing bundle qualification and verify the displayed recipe in a fresh workspace with semantic output assertions. +- [x] 3.3 Add a MoonBit guide explaining the unsupported Extism route, separate install/invoke selection and IR/envelope constraints; verify it links the separately dispatched Extism proposal/bead when available and presents no direct-invocation success claim without independent qualification. +- [x] 3.4 Add bounded producer/consumer probes for file interchange using supplied fixtures and pinned CLI identities; publish only verified directions, with checked refusals for incompatible IR/envelope/provenance cases. Verify the guide never equates file exchange with extension invocation or relies on invented runtime conversions. +- [x] 3.5 Record reusable host-recipe qualification evidence, input/command hashes, artifact digests, platform and checker identity; verify changed pins/recipes require fresh results, missing prerequisites cannot pass and reuse requires matching relevant inputs. Document rerun and support-status update commands alongside each guide. + +## 4. Validate and publish documentation through scoped CI (morphir-ukvn.9.4) + +- [x] 4.1 Extend `scripts/ci_changes.py` and its focused tests for docs/static, examples/referenced APIs and host-recipe checks; verify prose changes avoid unrelated compiler builds, candidate examples reuse compatible wheel producers, unknown/full changes remain conservative, and failures/cancellation/missing/unexpected skips fail the aggregate. +- [x] 4.2 Add a reusable site build/check workflow and wire selected checks into main CI; verify PRs build and test without production privileges, both deployment bases pass and the complete candidate's required example/host evidence is available or explicitly blocks publication. +- [x] 4.3 Add trusted-main Pages publication using the exact checked site artifact and source/tool/package/evidence manifest; verify deploy depends on all required checks, missing or mismatched evidence refuses promotion, and deploy-only permissions and the github-pages environment are correctly scoped. +- [x] 4.4 Document deployment settings, authorized manual runs, pin updates, evidence retention and rollback; verify the instructions identify any required maintainer setting and can restore a retained verified artifact without rebuilding it. +- [x] 4.5 Run the complete candidate's site/browser/example/host checks and required code quality gates; record actual run identities, tool versions and representative timings, reassess the provisional CI design, and verify no supported recipe or published snippet lacks qualification. + +## Workflow follow-up + +- Obtain review and maintainer approval before merging or deploying. If Pages settings are unavailable, present the verified artifact and exact required setting as the final approval step. +- After authorized main publication, verify public links, base paths, search and example downloads; retain the deployed source/artifact identity. +- Archive after implementation and review requirements are satisfied, repair the archived design asset paths, relink the epic spec id and close beads with commit/test evidence. Sync the shared tracker. diff --git a/packages/morphir-python-native/README.md b/packages/morphir-python-native/README.md index 6e18cc09..1f8a3c3a 100644 --- a/packages/morphir-python-native/README.md +++ b/packages/morphir-python-native/README.md @@ -1,5 +1,10 @@ # morphir-python-native +Read the [documentation site](https://finos.github.io/morphir-python/) for +[installation](https://finos.github.io/morphir-python/docs/installation/), +[API examples](https://finos.github.io/morphir-python/docs/python-compiler/) and +[CLI workflows](https://finos.github.io/morphir-python/docs/quickstart/). + The native compiler behind the `morphir-python` tools. It contains a private CPython extension module built from the Rust `morphir-python-binding` compiler with [PyO3](https://pyo3.rs) and [Maturin](https://www.maturin.rs). diff --git a/packages/morphir-tools/README.md b/packages/morphir-tools/README.md index 8131ba3b..8f3f2b7b 100644 --- a/packages/morphir-tools/README.md +++ b/packages/morphir-tools/README.md @@ -1,5 +1,10 @@ # Morphir Python +Read the [documentation site](https://finos.github.io/morphir-python/) for +[installation](https://finos.github.io/morphir-python/docs/installation/), +[API examples](https://finos.github.io/morphir-python/docs/python-compiler/) and +[CLI workflows](https://finos.github.io/morphir-python/docs/quickstart/). + CLI tools and extensions for [Morphir Python](https://github.com/finos/morphir-python). This package provides: diff --git a/packages/morphir-tools/pyproject.toml b/packages/morphir-tools/pyproject.toml index b9892451..f8c7de40 100644 --- a/packages/morphir-tools/pyproject.toml +++ b/packages/morphir-tools/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ [project.urls] Homepage = "https://github.com/finos/morphir-python" -Documentation = "https://morphir.finos.org" +Documentation = "https://finos.github.io/morphir-python/" Repository = "https://github.com/finos/morphir-python" Issues = "https://github.com/finos/morphir-python/issues" diff --git a/packages/morphir/README.md b/packages/morphir/README.md index ea65c282..8bbea999 100644 --- a/packages/morphir/README.md +++ b/packages/morphir/README.md @@ -1,5 +1,10 @@ # Morphir +Read the [documentation site](https://finos.github.io/morphir-python/) for +[installation](https://finos.github.io/morphir-python/docs/installation/), +[API examples](https://finos.github.io/morphir-python/docs/python-compiler/) and +[CLI workflows](https://finos.github.io/morphir-python/docs/quickstart/). + Python port of [Morphir](https://morphir.finos.org) - functional domain modeling for the enterprise. This is the core library package containing: diff --git a/packages/morphir/pyproject.toml b/packages/morphir/pyproject.toml index 911bc14d..b34bc3d5 100644 --- a/packages/morphir/pyproject.toml +++ b/packages/morphir/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [] [project.urls] Homepage = "https://github.com/finos/morphir-python" -Documentation = "https://morphir.finos.org" +Documentation = "https://finos.github.io/morphir-python/" Repository = "https://github.com/finos/morphir-python" Issues = "https://github.com/finos/morphir-python/issues" diff --git a/pyproject.toml b/pyproject.toml index ccc86744..ba215b80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ ignore = [ "scripts/bench.py" = ["E501"] # Compiler input: the supported subset has no TYPE_CHECKING blocks. "tests/fixtures/python/**/*" = ["TC001", "TC003"] +"examples/model/**/*" = ["TC001", "TC003", "D101", "D103"] "packages/morphir/src/morphir/ir/**/*.py" = ["D101", "D102", "D103", "UP006", "UP007", "UP035", "UP037", "UP046", "TC001", "TC003", "E501", "C408", "RUF005", "E402"] [tool.ruff.lint.pydocstyle] diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index 1321b138..8172e14d 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -22,7 +22,15 @@ if TYPE_CHECKING: from collections.abc import Iterable, Mapping -CATEGORIES = ("python", "native", "rust", "bundle") +CATEGORIES = ( + "python", + "native", + "rust", + "bundle", + "docs", + "docs-examples", + "docs-hosts", +) """``python``: lint, type check and Python tests; ``native``: the native wheel and its clean-install qualification; ``rust``: Rust format, lint, tests and the independent consumer; ``bundle``: the WASM extension bundle and its host check.""" @@ -35,33 +43,55 @@ "rust": ("rust",), "bundle": ("bundle",), "build": ("python", "native"), + "docs": ("docs", "docs-examples", "docs-hosts"), } """Each job runs when any of its categories is selected.""" # First match wins. Each rule names the categories a matching path selects. RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( + (".github/workflows/docs.yml", ("docs", "docs-examples", "docs-hosts")), + (".github/workflows/site-publish.yml", ("docs", "docs-examples", "docs-hosts")), + (".config/mise/tasks/site/*", ("docs",)), + (".config/mise/tasks/docs/*", ("docs", "docs-examples")), + ("site/integrations.json", ("docs", "docs-examples", "docs-hosts")), + ("site/evidence/*", ("docs", "docs-hosts")), + ("site/*", ("docs",)), + ("examples/extension.sh", ("docs", "docs-examples", "docs-hosts")), + ("examples/model/*", ("docs", "docs-examples", "docs-hosts")), + ("examples/*", ("docs", "docs-examples")), + ("scripts/docs_hosts.py", ("docs", "docs-examples", "docs-hosts")), + ("scripts/docs_*.py", ("docs", "docs-examples")), + ("scripts/site_*.py", ("docs", "docs-examples")), + ("tests/tools/test_python_docs.py", ("docs", "docs-examples")), + ("tests/unit/test_docs*.py", ("docs", "docs-examples")), + ("tests/unit/test_site*.py", ("docs", "docs-examples")), + ("docs/python-extension.md", ("docs", "docs-examples", "docs-hosts")), + ("docs/moonbit.md", ("docs", "docs-examples", "docs-hosts")), + ("docs/compatibility.md", ("docs", "docs-examples", "docs-hosts")), + ("docs/templates/*", ()), + ("docs/site.md", ("docs",)), + ("docs/site-foundation.md", ("docs",)), + ("docs/*", ("docs", "docs-examples")), (".github/*", CATEGORIES), (".config/mise/*", CATEGORIES), - ("rust/extension.toml", ("bundle",)), + ("rust/extension.toml", ("bundle", "docs", "docs-examples", "docs-hosts")), # Lint and type checks cover every script. - ("scripts/extension_bundle.py", ("python", "bundle")), + ("scripts/extension_bundle.py", ("python", "bundle", "docs", "docs-hosts")), ("rust/crates/morphir-python-binding/*", CATEGORIES), ("rust/crates/morphir-python-native/*", ("python", "native", "rust")), ("rust/*", CATEGORIES), - ("packages/morphir-python-native/*", ("python", "native")), + ("packages/morphir-python-native/*", ("python", "native", "docs", "docs-examples")), ("scripts/qualify_native.py", ("python", "native")), - ("tests/fixtures/python/*", ("python", "native", "bundle")), - ("packages/*", ("python", "native")), + ("tests/fixtures/python/*", ("python", "native", "bundle", "docs", "docs-hosts")), + ("packages/*", ("python", "native", "docs", "docs-examples")), ("tests/*", ("python", "native")), ("features/*", ("python", "native")), ("scripts/*", ("python", "native")), - ("pyproject.toml", ("python", "native")), - ("uv.lock", ("python", "native")), + ("pyproject.toml", ("python", "native", "docs", "docs-examples")), + ("uv.lock", ("python", "native", "docs", "docs-examples")), ("behave.ini", ("python",)), (".pre-commit-config.yaml", ("python",)), - ("docs/python-compiler.md", ("python",)), ("*.md", ()), - ("docs/*", ()), ("openspec/*", ()), ("LICENSE", ()), ("LICENSE.spdx", ()), @@ -70,7 +100,7 @@ ("skills-lock.json", ()), (".beads/*", ()), # Hatchling uses .gitignore to choose the files in each source archive. - (".gitignore", ("python", "native")), + (".gitignore", ("python", "native", "docs", "docs-examples")), (".gitattributes", ()), ) diff --git a/scripts/docs_check.py b/scripts/docs_check.py new file mode 100644 index 00000000..a6e90d9e --- /dev/null +++ b/scripts/docs_check.py @@ -0,0 +1,155 @@ +"""Qualify released and candidate documentation examples in isolated wheel installs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] + + +def run(command: list[str]) -> None: + """Execute a required qualification step and preserve its exit status.""" + subprocess.run(command, cwd=ROOT, check=True) + + +def examples(python: Path, group: str = "all") -> dict[str, Any]: + """Capture an installed example result from its own interpreter.""" + output = subprocess.check_output( + [str(python), str(ROOT / "scripts/docs_examples.py"), "test", "--group", group], + cwd=ROOT, + text=True, + ) + return dict(json.loads(output)) + + +def main() -> None: + """Build pure wheels once and qualify both released and development examples.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--native-wheel-dir", type=Path) + parser.add_argument("--report", type=Path, default=ROOT / ".dev/docs-examples.json") + args = parser.parse_args() + manifest = json.loads((ROOT / "examples/manifest.json").read_text()) + released = manifest["released"] + output = ROOT / ".dev/docs-wheels" + output.mkdir(parents=True, exist_ok=True) + # The only compiled wheel can be supplied by the existing CI producer. + for package in ("morphir", "morphir-tools"): + run( + [ + "uv", + "build", + str(ROOT / "packages" / package), + "--wheel", + "--out-dir", + str(output), + ] + ) + with tempfile.TemporaryDirectory(prefix="morphir-docs-wheels-") as scratch: + base = Path(scratch) + + def environment(name: str, requirements: list[str]) -> Path: + directory = base / name + run(["uv", "venv", "--python", sys.executable, str(directory)]) + python = directory / ( + "Scripts/python.exe" if sys.platform == "win32" else "bin/python" + ) + run( + [ + "uv", + "pip", + "install", + "--python", + str(python), + "--only-binary=:all:", + "--refresh", + *requirements, + ] + ) + return python + + released_python = environment( + "released", [f"{p}=={v}" for p, v in released.items()] + ) + released_report = examples(released_python) + library_python = environment("library", [f"morphir=={released['morphir']}"]) + library_report = examples(library_python, "library") + run( + [ + str(library_python), + "-I", + "-c", + "import importlib.util; " + "assert importlib.util.find_spec('morphir_python_native') is None", + ] + ) + candidate_wheels = [] + for package in ("morphir", "morphir-tools"): + import tomllib + + project = tomllib.loads( + (ROOT / "packages" / package / "pyproject.toml").read_text() + )["project"] + wheel_name = ( + project["name"].replace("-", "_") + + "-" + + project["version"] + + "-py3-none-any.whl" + ) + candidate_wheels.append(str(output / wheel_name)) + native = [] + if args.native_wheel_dir: + native = [str(p.absolute()) for p in args.native_wheel_dir.glob("*.whl")] + if len(native) != 1: + parser.error("Expected exactly one qualified native wheel") + candidate_python = environment("candidate", candidate_wheels + native) + candidate_report = examples(candidate_python) + run( + [ + "uv", + "pip", + "install", + "--python", + str(candidate_python), + "pytest==9.0.2", + "mypy==1.19.1", + "pyright==1.1.408", + ] + ) + run([str(candidate_python), str(ROOT / "scripts/docs_examples.py"), "check"]) + run( + [ + str(candidate_python), + "-m", + "pytest", + "tests/tools/test_python_docs.py", + "tests/unit/test_docs_examples.py", + "tests/unit/test_docs_hosts.py", + ] + ) + report = { + "state": "succeeded", + "source": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "released": released_report, + "candidate": candidate_report, + "library": library_report, + "wheels": { + Path(p).name: hashlib.sha256(Path(p).read_bytes()).hexdigest() + for p in candidate_wheels + native + }, + } + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(args.report) + + +if __name__ == "__main__": + main() diff --git a/scripts/docs_examples.py b/scripts/docs_examples.py new file mode 100644 index 00000000..2ca63d6a --- /dev/null +++ b/scripts/docs_examples.py @@ -0,0 +1,271 @@ +"""Synchronize and verify the documentation's canonical installed examples.""" + +from __future__ import annotations + +import argparse +import inspect +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +MARKER = re.compile( + r".*?", re.S +) +REFERENCE = re.compile(r".*?", re.S) + + +def source_region(root: Path, name: str, region: str | None) -> str: + """Read a contained example file or a named source region.""" + path = (root / name).resolve() + if not path.is_relative_to((root / "examples").resolve()): + raise ValueError(f"Example source escapes examples/: {name}") + text = path.read_text() + if region: + match = re.search( + rf"^# docs:start {re.escape(region)}\n" + rf"(.*?)^# docs:end {re.escape(region)}$", + text, + re.M | re.S, + ) + if match is None: + raise ValueError(f"Missing region {name}#{region}") + text = match[1] + return text.rstrip() + "\n" + + +def reference(name: str) -> str: + """Describe actual public exports and command options without rich wrapping.""" + if name == "api": + from morphir_tools import python as api + + lines = [] + for symbol in api.__all__: + value = getattr(api, symbol) + if inspect.isfunction(value) or inspect.isclass(value): + try: + rendered = str(inspect.signature(value)) + except ValueError: + rendered = "" + lines.append(symbol + rendered) + else: + lines.append(symbol) + return "\n".join(lines) + "\n" + from typer.main import get_command + + from morphir_tools.cli import app + + root: Any = get_command(app) + command = ( + root.commands[name] + if name in ("compile", "generate") + else root.commands["ir"].commands[name] + ) + lines = [f"morphir-python {'ir ' if name in ('inspect', 'convert') else ''}{name}"] + for param in command.params: + flags = ", ".join(param.opts + getattr(param, "secondary_opts", [])) + required = " [required]" if param.required else "" + default = f" [default: {param.default}]" if param.default is not None else "" + lines.append(f"{flags}{required}{default}: {getattr(param, 'help', '') or ''}") + return "\n".join(lines) + "\n" + + +def synchronize(root: Path = ROOT, *, write: bool = False) -> list[str]: + """Update snippets/references, or report drift without modifying source pages.""" + changed = [] + for page in sorted((root / "docs").glob("*.md")): + original = page.read_text() + + def snippet(match: re.Match[str]) -> str: + name, region = match.groups() + language = "python" if name.endswith(".py") else "bash" + key = name + (f"#{region}" if region else "") + return ( + f"\n```{language}\n" + f"{source_region(root, name, region)}```\n" + ) + + updated = MARKER.sub(snippet, original) + updated = REFERENCE.sub( + lambda m: ( + f"\n```text\n" + f"{reference(m[1])}```\n" + ), + updated, + ) + if updated != original: + changed.append(page.relative_to(root).as_posix()) + if write: + page.write_text(updated) + return changed + + +def run_examples(python: Path, group: str = "all") -> dict[str, object]: + """Execute examples and CLI recipes outside the checkout with installed packages.""" + manifest = json.loads((ROOT / "examples/manifest.json").read_text()) + completed = [] + with tempfile.TemporaryDirectory(prefix="morphir-docs-") as scratch: + for item in manifest["examples"]: + if group != "all" and item["packages"] != group: + continue + result = subprocess.run( + [str(python), "-I", str(ROOT / item["source"])], + cwd=scratch, + capture_output=True, + text=True, + check=True, + ) + if result.stdout != item["stdout"]: + raise ValueError( + f"Unexpected output for {item['source']}: {result.stdout!r}" + ) + if list(Path(scratch).iterdir()): + raise ValueError(f"Example wrote unexpected files: {item['source']}") + completed.append(item["source"]) + if group != "library": + tools = Path(scratch) / "tools" + tools.mkdir() + for name in ("bash", "mkdir", "cat"): + executable_path = shutil.which(name) + if executable_path is None: + raise ValueError(f"Missing POSIX recipe tool: {name}") + (tools / name).symlink_to(executable_path) + environment = os.environ | { + "PATH": str(python.parent) + os.pathsep + str(tools) + } + if shutil.which("rustc", path=environment["PATH"]) is not None: + raise ValueError("Example environment must not expose Rust") + subprocess.run( + ["bash", str(ROOT / "examples/cli.sh")], + cwd=scratch, + env=environment, + check=True, + capture_output=True, + text=True, + ) + for name in ( + "model.json", + "rebuilt.json", + "model-v4.json", + "generated/acme/models.py", + "generated/acme/rules.py", + ): + if not (Path(scratch) / name).is_file(): + raise ValueError(f"CLI example did not create {name}") + # Compare decoded IR, not JSON formatting or source spelling. + compare = ( + "from pathlib import Path; " + "from morphir.ir.v3.codec import decode_versioned_distribution as d; " + "assert d(Path('model.json').read_text()) == " + "d(Path('rebuilt.json').read_text())" + ) + subprocess.run([str(python), "-I", "-c", compare], cwd=scratch, check=True) + behavior = ( + "import sys; sys.path.insert(0, 'generated'); " + "from acme.models import Approved, Pending; " + "from acme.rules import choose; " + "assert choose(True, Approved(3.0), Pending()) == Approved(3.0)" + ) + subprocess.run([str(python), "-I", "-c", behavior], cwd=scratch, check=True) + executable = python.parent / "morphir-python" + protected = Path(scratch) / "generated/acme/models.py" + before = protected.read_bytes() + failure = subprocess.run( + [ + str(executable), + "generate", + "model.json", + "--output-dir", + "generated", + ], + cwd=scratch, + capture_output=True, + text=True, + ) + if failure.returncode != 1 or protected.read_bytes() != before: + raise ValueError("Nonempty generation directory did not fail safely") + invalid = Path(scratch) / "invalid.py" + invalid.write_text("answer = 42\n") + model_before = (Path(scratch) / "model.json").read_bytes() + failure = subprocess.run( + [ + str(executable), + "compile", + str(invalid), + "--package", + "acme/example", + "--output", + "model.json", + ], + cwd=scratch, + capture_output=True, + text=True, + ) + if failure.returncode != 1 or "PY004" not in failure.stderr: + raise ValueError("Unsupported-source diagnostic changed") + if (Path(scratch) / "model.json").read_bytes() != model_before: + raise ValueError("Failed compilation changed existing IR output") + completed.append("examples/cli.sh") + packages = json.loads( + subprocess.check_output( + [ + str(python), + "-I", + "-c", + "import json,importlib.metadata as m; " + "print(json.dumps({d.metadata['Name']:d.version " + "for d in m.distributions()}))", + ], + text=True, + ) + ) + return {"state": "succeeded", "examples": completed, "packages": packages} + + +def archive_examples() -> None: + """Create a deterministic download from the same checked example files.""" + output = ROOT / "site/downloads/examples.zip" + output.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(output, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted((ROOT / "examples").rglob("*")): + if path.is_file() and "__pycache__" not in path.parts: + info = zipfile.ZipInfo( + path.relative_to(ROOT).as_posix(), (2026, 1, 1, 0, 0, 0) + ) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, path.read_bytes()) + + +def main() -> None: + """Synchronize source snippets or run installed learning workflows.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["sync", "check", "test", "archive"]) + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + parser.add_argument( + "--group", choices=["all", "compiler", "library"], default="all" + ) + args = parser.parse_args() + if args.command in ("sync", "check"): + changed = synchronize(write=args.command == "sync") + if changed: + print("\n".join(changed)) + if args.command == "check": + parser.exit( + 1, + "Run scripts/docs_examples.py sync and review the snippets.\n", + ) + elif args.command == "archive": + archive_examples() + else: + print(json.dumps(run_examples(args.python.absolute(), args.group), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/docs_hosts.py b/scripts/docs_hosts.py new file mode 100644 index 00000000..e2248d5c --- /dev/null +++ b/scripts/docs_hosts.py @@ -0,0 +1,364 @@ +"""Qualify the displayed host recipe and the pinned MoonBit envelope refusal.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts import extension_bundle as bundle # noqa: E402 + +PINS = ROOT / "site/integrations.json" +INPUTS: tuple[str, ...] = ( + "site/integrations.json", + "rust/extension.toml", + "scripts/docs_hosts.py", + "scripts/extension_bundle.py", + "examples/extension.sh", + "examples/model/acme/models.py", + "examples/model/acme/rules.py", + "docs/python-extension.md", + "docs/moonbit.md", + "docs/compatibility.md", +) +INPUTS += tuple( + p.relative_to(ROOT).as_posix() for p in sorted(bundle.SAMPLE.rglob("*.py")) +) + + +def sha(data: bytes) -> str: + """Hash source or artifact bytes.""" + return hashlib.sha256(data).hexdigest() + + +def input_hashes(root: Path = ROOT) -> dict[str, str]: + """Bind evidence to the recipe, pins, fixtures and checker implementation.""" + return {name: sha((root / name).read_bytes()) for name in INPUTS} + + +def verify_evidence(report: dict[str, Any], root: Path = ROOT) -> None: + """Refuse missing, failed or stale qualification evidence.""" + if report.get("state") != "succeeded" or report.get("inputs") != input_hashes(root): + raise ValueError( + "Host evidence is missing, failed or stale; run docs_hosts.py qualify" + ) + if not report.get("rust_recipe") or not report.get("moonbit_refusals"): + raise ValueError("Host evidence lacks a required result") + for version in ("3", "4"): + result = report["moonbit_refusals"].get(version, {}) + if result.get("error") != "extension.unsupported_model_envelope": + raise ValueError("Host evidence lacks the qualified envelope refusal") + + +def download(url: str, expected: str, destination: Path) -> bytes: + """Cache only bytes that match the declared SHA-256.""" + if destination.is_file() and sha(destination.read_bytes()) == expected: + return destination.read_bytes() + with urllib.request.urlopen(url, timeout=120) as response: + data: bytes = response.read() + if sha(data) != expected: + raise ValueError(f"SHA-256 mismatch: {url}") + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(data) + return data + + +def prepare() -> tuple[Path, Path]: + """Acquire the declared Rust CLI and released Python bundle.""" + pins = json.loads(PINS.read_text()) + host, archive_digest = bundle.morphir_cli() + version = pins["rust_host"]["version"] + if bundle.declaration()["host"]["version"] != version: + raise ValueError("Host declaration and documentation pins disagree") + target = host.parent.name.removeprefix(f"morphir-{version}-") + archive_name = f"morphir-{version}-{target}.tgz" + data = download( + bundle.declaration()["host"]["release"] + "/" + archive_name, + archive_digest, + ROOT / ".dev/docs-hosts" / archive_name, + ) + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + member = next(m for m in archive.getmembers() if Path(m.name).name == "morphir") + stream = archive.extractfile(member) + if stream is None or sha(stream.read()) != sha(host.read_bytes()): + raise ValueError("Cached host does not match its verified release archive") + identity = subprocess.check_output([str(host), "--version"], text=True) + if f"morphir {version}" not in identity: + raise ValueError("Wrong Morphir provider/version") + spec = pins["python_bundle"] + directory = ROOT / ".dev/docs-hosts" / f"python-{spec['version']}" + for name, expected in spec["files"].items(): + destination = directory / ( + "release.json" if name.endswith(".release.json") else name + ) + download(spec["release"] + "/" + name, expected, destination) + bundle.receipt(directory, version=spec["version"]) + return host, directory + + +def qualify_rust(host: Path, directory: Path) -> dict[str, Any]: + """Run the exact displayed installation recipe, then existing offline checks.""" + with tempfile.TemporaryDirectory(prefix="docs-rust-host-") as scratch: + work = Path(scratch) + shutil.copytree(ROOT / "examples/model", work / "src") + result = subprocess.run( + ["bash", str(ROOT / "examples/extension.sh")], + cwd=work, + env=os.environ + | {"MORPHIR_HOST": str(host), "PYTHON_BUNDLE": str(directory)}, + capture_output=True, + text=True, + check=True, + timeout=120, + ) + first = json.loads((work / "compiled/morphir-ir.json").read_text()) + again = json.loads((work / "host-rebuilt/morphir-ir.json").read_text()) + if first != again: + raise ValueError("Displayed host recipe changed IR across its round trip") + if not (work / "host-generated/acme/rules.py").is_file(): + raise ValueError( + "Displayed host recipe did not generate its documented module" + ) + return { + "recipe": "examples/extension.sh", + "exit_code": result.returncode, + "host_sha256": sha(host.read_bytes()), + "offline": bundle.host_check( + directory, + version=json.loads(PINS.read_text())["python_bundle"]["version"], + ), + } + + +def moonbit_source() -> Path: + """Build the exact source-pinned CLI in this repository's disposable tool cache.""" + spec = json.loads(PINS.read_text())["moonbit"] + cache = ROOT / ".dev/docs-hosts" + data = download( + f"https://codeload.github.com/finos/morphir-moonbit/tar.gz/{spec['revision']}", + spec["archive_sha256"], + cache / "moonbit-source.tar.gz", + ) + source: Path = cache / ("moonbit-" + spec["revision"][:12]) + # Always extract fresh source; a cache of a compiled CLI is not source evidence. + if source.exists(): + shutil.rmtree(source) + source.mkdir(parents=True) + with tarfile.open(fileobj=io.BytesIO(data)) as archive: + archive.extractall(source, filter="data") + source = next(source.iterdir()) + subprocess.run( + ["mise", "trust", ".config/mise/config.toml"], + cwd=source, + check=True, + capture_output=True, + text=True, + timeout=30, + ) + tool = "http:moonbit@" + str(spec["toolchain"]) + subprocess.run( + ["mise", "install", tool], + cwd=source, + check=True, + capture_output=True, + text=True, + timeout=180, + ) + moon_home = Path( + subprocess.check_output(["mise", "where", tool], cwd=source, text=True).strip() + ) + moon = moon_home / "bin/moon" + environment = os.environ | { + "MOON_HOME": str(moon_home), + "PATH": str(moon_home / "bin") + os.pathsep + os.environ["PATH"], + } + for arguments in ( + ["update"], + ["build", "apps/morphir", "--target", "js", "--release"], + ): + subprocess.run( + [str(moon), *arguments], + cwd=source, + env=environment, + check=True, + capture_output=True, + text=True, + timeout=300, + ) + return source + + +def qualify_moonbit() -> dict[str, Any]: + """Probe plain v3/v4 IR at the real portable CLI boundary before execution.""" + from morphir.ir.v3 import codec as v3_codec + from morphir.ir.v4 import codec as v4_codec + from morphir_tools.python import ( + CompiledV3, + CompiledV4, + IrVersion, + SourceDocument, + compile_sources, + ) + + source = moonbit_source() + binary = source / "_build/js/release/build/morphir/morphir/morphir.js" + modules = [ + "node-mep-cli.mjs", + "node-mep-stdio.mjs", + "node-mep-adapter.mjs", + "node-mep-worker.mjs", + "mep-adapter.mjs", + "mep-frames.mjs", + "wasm-memory.mjs", + "node-artifacts.mjs", + "artifacts.mjs", + "assembly.mjs", + "index.mjs", + "dist/bridge.mjs", + ] + pin: dict[str, Any] = { + "contractVersion": "0.1.0-draft.1", + "binding": "core-wasm-process", + "executionProfile": { + "id": "morphir/mep-bytes-v1/v1-draft", + "sha256": ( + "6ed5e96fa9c7f7dcaa179efc88e517307463cfeddc5e2eeae854f0395e7a4ec6" + ), + }, + "runtime": { + "name": "node", + "version": "v24.22.1", + "executable": "/not-executed/node", + "sha256": "0" * 64, + }, + "host": { + "root": "/not-executed/host", + "modules": dict.fromkeys(modules, "0" * 64), + }, + "artifact": { + "directory": "/not-executed/guest", + "wasmSha256": "0" * 64, + "manifestSha256": "0" * 64, + }, + "origin": "Negative envelope probe: execution must never start", + } + results = {} + sources = [ + SourceDocument(p.relative_to(ROOT / "examples/model").as_posix(), p.read_text()) + for p in sorted((ROOT / "examples/model").rglob("*.py")) + ] + with tempfile.TemporaryDirectory(prefix="docs-moonbit-") as scratch: + work = Path(scratch) + (work / "pin.json").write_text(json.dumps(pin)) + for version in IrVersion: + compiled = compile_sources( + sources, package="acme/example", ir_version=version + ) + if isinstance(compiled, CompiledV3): + document = v3_codec.encode_versioned_distribution(compiled.distribution) + elif isinstance(compiled, CompiledV4): + document = v4_codec.encode_versioned_distribution(compiled.distribution) + else: + raise ValueError("Interchange fixture failed compilation") + (work / "plain.json").write_text(json.dumps(document)) + command = [ + "node", + str(binary), + "extension", + "generate", + "plain.json", + "--extension-pin", + "pin.json", + "--extension-helper", + "/not-executed/helper.mjs", + "--output", + "generated", + "--json", + ] + result = subprocess.run( + command, cwd=work, text=True, capture_output=True, timeout=20 + ) + report = json.loads(result.stdout) + if ( + result.returncode != 2 + or report.get("error") != "extension.unsupported_model_envelope" + or (work / "generated").exists() + ): + raise ValueError( + f"MoonBit envelope refusal changed: {result.stdout} {result.stderr}" + ) + results[version.value] = { + "exit_code": result.returncode, + "error": report["error"], + "input_sha256": sha((work / "plain.json").read_bytes()), + "command": [ + arg.replace(str(source), "PINNED_SOURCE") for arg in command + ], + } + return { + "revision": json.loads(PINS.read_text())["moonbit"]["revision"], + "cli_sha256": sha(binary.read_bytes()), + "results": results, + } + + +def main() -> None: + """Prepare, qualify or verify reusable host compatibility evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["prepare", "qualify", "verify"]) + parser.add_argument( + "--report", type=Path, default=ROOT / ".dev/docs-hosts/report.json" + ) + args = parser.parse_args() + try: + if args.command == "verify": + verify_evidence(json.loads(args.report.read_text())) + print("Host evidence matches all relevant inputs") + return + host, directory = prepare() + if args.command == "prepare": + print(f"MORPHIR_HOST={host}\nPYTHON_BUNDLE={directory}") + return + rust = qualify_rust(host, directory) + moonbit = qualify_moonbit() + report = { + "schema_version": 1, + "state": "succeeded", + "inputs": input_hashes(), + "platform": platform.platform(), + "python": sys.version, + "working_tree_dirty": bool( + subprocess.check_output( + ["git", "status", "--porcelain"], cwd=ROOT, text=True + ) + ), + "git_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "rust_recipe": rust, + "moonbit_refusals": moonbit["results"], + "moonbit": moonbit, + } + verify_evidence(report) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(args.report) + except (ValueError, OSError, subprocess.SubprocessError, bundle.BundleError) as exc: + parser.exit(1, f"host qualification: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/extension_bundle.py b/scripts/extension_bundle.py index 560aac84..53f9699e 100644 --- a/scripts/extension_bundle.py +++ b/scripts/extension_bundle.py @@ -170,10 +170,10 @@ def package(wasm: Path, output: Path, git_commit: str | None) -> dict[str, objec return receipt(output) -def receipt(bundle: Path) -> dict[str, object]: +def receipt(bundle: Path, *, version: str | None = None) -> dict[str, object]: """Check a bundle's files, digest and claims, and describe them.""" extension = declaration()["extension"] - version = crate_version() + version = crate_version() if version is None else version artifact = f"{extension['artifact']}-{version}.wasm" names = sorted(p.name for p in bundle.iterdir()) if names != sorted([artifact, f"{artifact}.sha256", "release.json"]): @@ -258,12 +258,12 @@ def morphir_cli() -> tuple[Path, str]: return binary, expected -def host_check(bundle: Path) -> dict[str, object]: +def host_check(bundle: Path, *, version: str | None = None) -> dict[str, object]: """Install the bundle with the pinned CLI and round trip offline.""" from morphir_tools.python import native_compiler bundle = bundle.resolve() - described = receipt(bundle) + described = receipt(bundle, version=version) cli, cli_digest = morphir_cli() with tempfile.TemporaryDirectory(prefix="python-bundle-host-") as scratch: work = Path(scratch) @@ -334,7 +334,9 @@ def morphir(*args: str, offline: bool = False) -> str: ) results[f"v{tag}"] = { "modules": files, - "agrees_with_python_api": agrees_with_api(first, tag), + "agrees_with_python_api": agrees_with_api( + first, tag, compiler_version=str(described["version"]) + ), } return { **described, @@ -347,7 +349,9 @@ def morphir(*args: str, offline: bool = False) -> str: } -def agrees_with_api(document: dict[str, Any], tag: str) -> bool: +def agrees_with_api( + document: dict[str, Any], tag: str, *, compiler_version: str | None = None +) -> bool: """Compare host output with the Python API's model for the same sources.""" from morphir.ir.v3 import codec as v3_codec from morphir.ir.v4 import codec as v4_codec @@ -360,10 +364,11 @@ def agrees_with_api(document: dict[str, Any], tag: str) -> bool: ) compiler = native_compiler() - if compiler.compiler_version != crate_version(): + expected_version = crate_version() if compiler_version is None else compiler_version + if compiler.compiler_version != expected_version: raise BundleError( f"API compiler {compiler.compiler_version} differs from " - f"bundle {crate_version()}" + f"bundle {expected_version}" ) sources = [ SourceDocument(p.relative_to(SAMPLE).as_posix(), p.read_text()) diff --git a/scripts/site_release.py b/scripts/site_release.py new file mode 100644 index 00000000..4adabf6f --- /dev/null +++ b/scripts/site_release.py @@ -0,0 +1,129 @@ +"""Bind the checked site bytes and required evidence to a deployable source revision.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from scripts.docs_hosts import verify_evidence # noqa: E402 + + +def tree_hashes(directory: Path) -> dict[str, str]: + """Identify every published file except the manifest that contains these hashes.""" + hashes = {} + for path in sorted(directory.rglob("*")): + if path.is_symlink(): + raise ValueError("Site artifact contains a symlink") + if path.is_file() and path != directory / "site-evidence.json": + hashes[path.relative_to(directory).as_posix()] = hashlib.sha256( + path.read_bytes() + ).hexdigest() + return hashes + + +def producer(run: dict[str, Any], repository: str) -> tuple[str, int]: + """Accept artifacts only from successful CI runs of this repository's main.""" + if ( + run.get("conclusion") != "success" + or run.get("event") not in ("push", "workflow_dispatch") + or run.get("head_branch") != "main" + or run.get("head_repository", {}).get("full_name") != repository + or run.get("path") != ".github/workflows/ci.yml" + ): + raise ValueError("Untrusted or unsuccessful site artifact producer") + source = str(run.get("head_sha", "")) + if len(source) != 40 or any(c not in "0123456789abcdef" for c in source): + raise ValueError("Invalid producer source revision") + return source, int(run["id"]) + + +def verify(directory: Path, source: str, *, deploy: bool) -> dict[str, Any]: + """Reject altered bytes, mismatched candidates or absent deployment evidence.""" + report: dict[str, Any] = json.loads((directory / "site-evidence.json").read_text()) + if report.get("source") != source: + raise ValueError("Site candidate does not match the selected source revision") + if report.get("state") != "succeeded" or report.get("files") != tree_hashes( + directory + ): + raise ValueError("Site artifact differs from the checked output") + if deploy and report.get("eligible_for_deploy") is not True: + raise ValueError("Site artifact lacks required deployment evidence") + return report + + +def seal(directory: Path, examples: Path | None, hosts: Path | None) -> None: + """Record the exact checked output and qualification results without rebuilding.""" + source = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip() + build = json.loads((directory / "telescope-build.json").read_text()) + pins = json.loads((ROOT / "site/toolchain.json").read_text()) + if build.get("toolkit_version") != pins["version"]: + raise ValueError("Telescope output does not match the pinned tool") + checks: dict[str, Any] = {} + if examples: + result = json.loads(examples.read_text()) + if result.get("state") != "succeeded" or result.get("source") != source: + raise ValueError("Example evidence does not match the site candidate") + for group in ("released", "candidate", "library"): + if result.get(group, {}).get("state") != "succeeded": + raise ValueError(f"Required example group did not succeed: {group}") + checks["examples"] = result + if hosts: + result = json.loads(hosts.read_text()) + verify_evidence(result) + checks["hosts"] = result + report = { + "schema_version": 1, + "state": "succeeded", + "source": source, + "eligible_for_deploy": bool(examples and hosts), + "tool": build, + "checks": checks, + "files": tree_hashes(directory), + } + (directory / "site-evidence.json").write_text(json.dumps(report, indent=2) + "\n") + verify(directory, source, deploy=bool(examples and hosts)) + + +def main() -> None: + """Seal a checked build or verify an immutable candidate before promotion.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["seal", "verify", "producer"]) + parser.add_argument("--site", type=Path, default=ROOT / "_site") + parser.add_argument("--source") + parser.add_argument("--examples", type=Path) + parser.add_argument("--hosts", type=Path) + parser.add_argument("--deploy", action="store_true") + parser.add_argument("--run-json", type=Path) + parser.add_argument("--repository") + args = parser.parse_args() + try: + if args.command == "producer": + if not args.run_json or not args.repository: + parser.error("producer requires --run-json and --repository") + source, run_id = producer( + json.loads(args.run_json.read_text()), args.repository + ) + print(f"source={source}\nrun={run_id}") + return + if args.command == "seal": + seal(args.site, args.examples, args.hosts) + else: + if not args.source: + parser.error("verify requires --source") + verify(args.site, args.source, deploy=args.deploy) + print("Site artifact and evidence verified") + except (OSError, ValueError, subprocess.SubprocessError) as exc: + parser.exit(1, f"site publication: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/site_tool.py b/scripts/site_tool.py new file mode 100644 index 00000000..202af6ab --- /dev/null +++ b/scripts/site_tool.py @@ -0,0 +1,154 @@ +"""Install the pinned Telescope release and run its stock consumer commands.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +PINS = ROOT / "site/toolchain.json" + + +class SiteError(Exception): + """A site tool prerequisite or release verification failed.""" + + +def digest(path: Path) -> str: + """Return the SHA-256 digest of a file.""" + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def install_archive( + archive: Path, destination: Path, expected: str, version: str +) -> Path: + """Install a verified native release without trusting its archive paths.""" + if digest(archive) != expected: + raise SiteError(f"SHA-256 mismatch for {archive.name}") + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as scratch: + staged = Path(scratch) / "install" + staged.mkdir() + with tarfile.open(archive) as release: + binaries = [ + m for m in release.getmembers() if Path(m.name).name == "telescope" + ] + if len(binaries) != 1 or not binaries[0].isfile(): + raise SiteError("Release must contain one regular telescope executable") + stream = release.extractfile(binaries[0]) + if stream is None: + raise SiteError("Cannot read release executable") + binary = staged / "telescope" + binary.write_bytes(stream.read()) + binary.chmod(0o755) + actual = subprocess.check_output([str(binary), "--version"], text=True).strip() + if actual != version: + raise SiteError( + f"Telescope version mismatch: expected {version}, got {actual}" + ) + (staged / "install.json").write_text( + json.dumps( + { + "version": version, + "archive_sha256": expected, + "binary_sha256": digest(binary), + }, + indent=2, + ) + + "\n" + ) + backup = destination.with_name(destination.name + ".previous") + if backup.exists(): + shutil.rmtree(backup) + if destination.exists(): + destination.rename(backup) + try: + staged.rename(destination) + except OSError: + if backup.exists(): + backup.rename(destination) + raise + if backup.exists(): + shutil.rmtree(backup) + return destination / "telescope" + + +def setup() -> Path: + """Download and verify the release for this qualified host platform.""" + config = json.loads(PINS.read_text()) + machine = {"aarch64": "arm64", "x86_64": "x64"}.get( + platform.machine(), platform.machine() + ) + target = f"{platform.system().lower()}-{machine}" + release = config["platforms"].get(target) + if release is None: + raise SiteError(f"Unqualified Telescope platform: {target}; see docs/site.md") + destination = ROOT / ".dev/tools" / f"telescope-{config['version']}-{target}" + binary = destination / "telescope" + receipt = destination / "install.json" + if binary.is_file() and receipt.is_file(): + installed = json.loads(receipt.read_text()) + if installed.get("archive_sha256") == release["sha256"] and installed.get( + "binary_sha256" + ) == digest(binary): + return binary + with tempfile.TemporaryDirectory(prefix="telescope-download-") as scratch: + archive = Path(scratch) / "release.tar.gz" + with urllib.request.urlopen(release["url"], timeout=120) as response: + archive.write_bytes(response.read()) + return install_archive( + archive, destination, release["sha256"], config["version"] + ) + + +def main(argv: list[str] | None = None) -> None: + """Run a reproducible site command without installing the Rust toolchain.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["setup", "build", "check", "serve"]) + parser.add_argument("--base", default="/morphir-python/") + parser.add_argument("--out", default="_site") + parser.add_argument("--port", type=int, default=4173) + args = parser.parse_args(argv) + try: + binary = setup() + if args.command == "setup": + print(binary) + return + if args.command == "build": + subprocess.run( + [sys.executable, str(ROOT / "scripts/docs_examples.py"), "archive"], + check=True, + ) + command = [ + str(binary), + args.command, + "--config", + "site/site.json", + "--out", + args.out, + "--base", + args.base, + ] + if args.command == "check": + command.extend(["--format", "json"]) + if args.command == "serve": + command.extend(["--port", str(args.port)]) + os.chdir(ROOT) + os.execv(binary, command) + subprocess.run(command, cwd=ROOT, check=True) + except (SiteError, OSError, subprocess.CalledProcessError) as exc: + parser.exit(1, f"site: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/site/browser/package-lock.json b/site/browser/package-lock.json new file mode 100644 index 00000000..4cbd7f7c --- /dev/null +++ b/site/browser/package-lock.json @@ -0,0 +1,58 @@ +{ + "name": "morphir-python-site-checks", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "morphir-python-site-checks", + "devDependencies": { + "@playwright/test": "1.64.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/site/browser/package.json b/site/browser/package.json new file mode 100644 index 00000000..aecc2d1b --- /dev/null +++ b/site/browser/package.json @@ -0,0 +1,7 @@ +{ + "name": "morphir-python-site-checks", + "private": true, + "type": "module", + "scripts": {"test": "node run.mjs"}, + "devDependencies": {"@playwright/test": "1.64.1"} +} diff --git a/site/browser/playwright.config.mjs b/site/browser/playwright.config.mjs new file mode 100644 index 00000000..4bdb7608 --- /dev/null +++ b/site/browser/playwright.config.mjs @@ -0,0 +1,21 @@ +import { defineConfig } from '@playwright/test'; +import { fileURLToPath } from 'node:url'; +const root = fileURLToPath(new URL('../../', import.meta.url)); +export default defineConfig({ + testDir: '.', + testMatch: '*.spec.mjs', + outputDir: '../../.dev/site-browser/test-results', + reporter: [['list'], ['json', { outputFile: '../../.dev/site-browser/results.json' }]], + workers: 2, + forbidOnly: !!process.env.CI, + retries: 0, + use: { browserName: 'chromium', trace: 'retain-on-failure' }, + projects: [ + { name: 'pages', use: { baseURL: 'http://127.0.0.1:4173' }, metadata: { base: '/morphir-python/' } }, + { name: 'root', use: { baseURL: 'http://127.0.0.1:4174' }, metadata: { base: '/' } }, + ], + webServer: [ + { command: 'python3 scripts/site_tool.py serve', cwd: root, url: 'http://127.0.0.1:4173/morphir-python/', reuseExistingServer: false }, + { command: 'python3 scripts/site_tool.py serve --base / --out .dev/site-root --port 4174', cwd: root, url: 'http://127.0.0.1:4174/', reuseExistingServer: false }, + ], +}); diff --git a/site/browser/run.mjs b/site/browser/run.mjs new file mode 100644 index 00000000..523a006a --- /dev/null +++ b/site/browser/run.mjs @@ -0,0 +1,12 @@ +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +const root = fileURLToPath(new URL('../../', import.meta.url)); +for (const args of [ + ['scripts/site_tool.py', 'build'], + ['scripts/site_tool.py', 'build', '--base', '/', '--out', '.dev/site-root'], +]) { + const result = spawnSync('python3', args, { cwd: root, stdio: 'inherit' }); + if (result.status !== 0) process.exit(result.status ?? 1); +} +const result = spawnSync(process.execPath, ['node_modules/@playwright/test/cli.js', 'test'], { stdio: 'inherit' }); +process.exit(result.status ?? 1); diff --git a/site/browser/site.spec.mjs b/site/browser/site.spec.mjs new file mode 100644 index 00000000..0627a1eb --- /dev/null +++ b/site/browser/site.spec.mjs @@ -0,0 +1,58 @@ +import { test, expect } from '@playwright/test'; +import { fileURLToPath } from 'node:url'; + +for (const width of [1440, 360]) for (const theme of ['light', 'dark']) { + test(`read and navigate at ${width}px in ${theme}`, async ({ page }, info) => { + const base = info.project.metadata.base; + await page.setViewportSize({ width, height: 900 }); + await page.emulateMedia({ colorScheme: theme }); + await page.goto(base + 'docs/quickstart/'); + await expect(page.getByRole('heading', { name: 'A Python model round trip', exact: true })).toBeVisible(); + const size = await page.evaluate(() => ({ page: document.documentElement.scrollWidth, screen: innerWidth })); + expect(size.page).toBeLessThanOrEqual(size.screen + 1); + await page.keyboard.press('Tab'); + const skip = page.getByRole('link', { name: 'Skip to content' }); + await expect(skip).toBeFocused(); + await expect(skip).not.toHaveCSS('outline-style', 'none'); + if (width === 360) { + const menu = page.getByRole('button', { name: 'Menu', exact: true }); + await menu.focus(); + await page.keyboard.press('Enter'); + await expect(page.getByRole('navigation', { name: 'Main', exact: true })).toBeVisible(); + await expect(menu).toHaveAttribute('aria-expanded', 'true'); + } + await page.screenshot({ path: fileURLToPath(new URL(`../../.dev/site-browser/${info.project.name}-${width}-${theme}.png`, import.meta.url)), fullPage: true }); + }); +} + +test('search finds the checked API guide and excludes development artifacts', async ({ page }, info) => { + const base = info.project.metadata.base; + await page.goto(base); + const search = page.getByRole('searchbox', { name: 'Search the site' }); + await page.keyboard.press('/'); + await expect(search).toBeFocused(); + await search.fill('SourceDocument'); + await expect(page.locator('.search-results a').first()).toBeVisible(); + const hrefs = await page.locator('.search-results a').evaluateAll(nodes => nodes.map(n => n.getAttribute('href'))); + expect(hrefs.every(href => href.startsWith(base + 'docs/'))).toBe(true); + expect(hrefs.some(href => href.includes('python-compiler') || href.includes('api-reference'))).toBe(true); + await search.fill('Privatetemplatetoken'); + await expect(page.locator('.search-empty')).toHaveText('No results'); + const index = await (await page.request.get(base + 'search-index.json')).text(); + expect(index).not.toMatch(/docs\/templates|docs\/superpowers|openspec\/changes/); +}); + +test('copy preserves canonical code, and the example archive is downloadable', async ({ page, context }, info) => { + const base = info.project.metadata.base; + await context.grantPermissions(['clipboard-read', 'clipboard-write']); + await page.goto(base + 'docs/quickstart/'); + const source = await page.locator('pre code').first().textContent(); + const copy = page.getByRole('button', { name: 'Copy code', exact: true }).first(); + await copy.focus(); + await page.keyboard.press('Enter'); + await expect(copy).toHaveAttribute('data-state', 'copied'); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe(source.trimEnd()); + const download = await page.request.get(base + 'assets/downloads/examples.zip'); + expect(download.ok()).toBe(true); + expect((await download.body()).subarray(0, 2).toString()).toBe('PK'); +}); diff --git a/site/evidence/hosts.json b/site/evidence/hosts.json new file mode 100644 index 00000000..9532c12d --- /dev/null +++ b/site/evidence/hosts.json @@ -0,0 +1,152 @@ +{ + "schema_version": 1, + "state": "succeeded", + "inputs": { + "site/integrations.json": "d83a529d8d4d59c7505892f93cb7e991939e38174892cd0f11f2172c090b3578", + "rust/extension.toml": "6fe92784006cb757c9d6d1e27c1869b2ac5d4ecda4753644d9d8d202b9d7ff5d", + "scripts/docs_hosts.py": "791c2e220217e8931e1c090749987a17e7787fcdbba35443d83b3e234f8b7cc6", + "scripts/extension_bundle.py": "2a2937f2274db14cfee54dd24616045f55592cefe7c99611aa73098fb44cbcd6", + "examples/extension.sh": "bbe0a0cf27070b888439f57a9e495eb75506b8fafbafd259b1a8512bcf1af35b", + "examples/model/acme/models.py": "0f57ac6f4ffd2524373ec9ba3df079f23291ce42ca7f7700e4df50fad9d43b50", + "examples/model/acme/rules.py": "cf475c980096c8645550c4c53c5e6343e3a584f2a19cba26c16cf1291ce235c2", + "docs/python-extension.md": "47969c1d2c7de272c6aa275ff0a558b1efc8778556c2796e0108c0a31ff8af4a", + "docs/moonbit.md": "574f1e0129975dc10ea68882b48dda07ced36f429b2a30746c6bc724e9f119c2", + "docs/compatibility.md": "3f5ef69142991edf9d30d23dce768d66cf0047f4b325c1f042166a5c2f9c29f8", + "tests/fixtures/python/loans/acme/models.py": "02b453935df7a03fd8e58c88616836d8f09e78868b9c66cae572c18852f6ccf4", + "tests/fixtures/python/loans/acme/rules.py": "6aa4573844f82910c06079ef1af11a56103dc48cbcdc142bfc23c3aa25f6162e" + }, + "platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O", + "python": "3.14.7 (main, Sep 1 2026, 14:05:28) [Clang 22.1.3 ]", + "working_tree_dirty": true, + "git_head": "a55985d32c24987b1ae790ea05acd4bc0a39deb1", + "rust_recipe": { + "recipe": "examples/extension.sh", + "exit_code": 0, + "host_sha256": "e610796e758c4f2b8925228082efd78f9b9d46c45ee22b6e393d9b76b8e807e2", + "offline": { + "bundle": { + "morphir-python-binding-0.4.2.wasm": "10acafa0f0282c5240e9620a5f2bb21629cc7580b9add528e7b77c7e54db22b5", + "morphir-python-binding-0.4.2.wasm.sha256": "64f60ff9e58e6cbe2f3323e1c375cef77be7db81b9879974f943918a92a42634", + "release.json": "72770f73f05a72535fd8c86ad4629175019037441a758b827c1ba47cc9a52ebd" + }, + "wasm_sha256": "10acafa0f0282c5240e9620a5f2bb21629cc7580b9add528e7b77c7e54db22b5", + "extension": "morphir-python", + "version": "0.4.2", + "git_commit": "1f547d17f648d5b398862f9dfceb05b6d463cb3f", + "packager_rev": "0aa60eb799d46f8a317ec47abef4a7a4e61dc8ac", + "capabilities": [ + "backend", + "frontend" + ], + "multiDocument": true, + "host": { + "morphir": "0.4.0-beta.10", + "sha256": "98635ec74fbdc3c2cef1d4ae184bb1c5ed9e2960575f3d8abb4b97e0e4da363f" + }, + "python_api": { + "native_version": "0.1.1", + "compiler_version": "0.4.2" + }, + "offline_round_trips": { + "v3": { + "modules": [ + "acme/models.py", + "acme/rules.py" + ], + "agrees_with_python_api": true + }, + "v4": { + "modules": [ + "acme/models.py", + "acme/rules.py" + ], + "agrees_with_python_api": true + } + } + } + }, + "moonbit_refusals": { + "3": { + "exit_code": 2, + "error": "extension.unsupported_model_envelope", + "input_sha256": "a5f914c2a1af4ca4426172992dd853492b3bd07a07e6f983a5f58ae3f9d54802", + "command": [ + "node", + "PINNED_SOURCE/_build/js/release/build/morphir/morphir/morphir.js", + "extension", + "generate", + "plain.json", + "--extension-pin", + "pin.json", + "--extension-helper", + "/not-executed/helper.mjs", + "--output", + "generated", + "--json" + ] + }, + "4": { + "exit_code": 2, + "error": "extension.unsupported_model_envelope", + "input_sha256": "758f39622c5d60ec42da0de3756bfc8186b2d306b362bcb3315c51b81387aabe", + "command": [ + "node", + "PINNED_SOURCE/_build/js/release/build/morphir/morphir/morphir.js", + "extension", + "generate", + "plain.json", + "--extension-pin", + "pin.json", + "--extension-helper", + "/not-executed/helper.mjs", + "--output", + "generated", + "--json" + ] + } + }, + "moonbit": { + "revision": "bd3adc52e6b8f6368fdaee50f76535799732f8d6", + "cli_sha256": "1a61e2008d62b773ce3f993c00f55a5fc2e242b726e9a272048d3710c5a446ea", + "results": { + "3": { + "exit_code": 2, + "error": "extension.unsupported_model_envelope", + "input_sha256": "a5f914c2a1af4ca4426172992dd853492b3bd07a07e6f983a5f58ae3f9d54802", + "command": [ + "node", + "PINNED_SOURCE/_build/js/release/build/morphir/morphir/morphir.js", + "extension", + "generate", + "plain.json", + "--extension-pin", + "pin.json", + "--extension-helper", + "/not-executed/helper.mjs", + "--output", + "generated", + "--json" + ] + }, + "4": { + "exit_code": 2, + "error": "extension.unsupported_model_envelope", + "input_sha256": "758f39622c5d60ec42da0de3756bfc8186b2d306b362bcb3315c51b81387aabe", + "command": [ + "node", + "PINNED_SOURCE/_build/js/release/build/morphir/morphir/morphir.js", + "extension", + "generate", + "plain.json", + "--extension-pin", + "pin.json", + "--extension-helper", + "/not-executed/helper.mjs", + "--output", + "generated", + "--json" + ] + } + } + } +} diff --git a/site/integrations.json b/site/integrations.json new file mode 100644 index 00000000..5cb3e362 --- /dev/null +++ b/site/integrations.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "rust_host": { + "version": "0.4.0-beta.10", + "declaration": "rust/extension.toml" + }, + "python_bundle": { + "version": "0.4.2", + "release": "https://github.com/finos/morphir-python/releases/download/extension/python/v0.4.2", + "files": { + "morphir-python-binding-0.4.2.release.json": "72770f73f05a72535fd8c86ad4629175019037441a758b827c1ba47cc9a52ebd", + "morphir-python-binding-0.4.2.wasm": "10acafa0f0282c5240e9620a5f2bb21629cc7580b9add528e7b77c7e54db22b5", + "morphir-python-binding-0.4.2.wasm.sha256": "64f60ff9e58e6cbe2f3323e1c375cef77be7db81b9879974f943918a92a42634" + } + }, + "moonbit": { + "repository": "https://github.com/finos/morphir-moonbit", + "revision": "bd3adc52e6b8f6368fdaee50f76535799732f8d6", + "toolchain": "0.10.16+b1a0e846e", + "expected_plain_ir_refusal": "extension.unsupported_model_envelope", + "archive_sha256": "a4975700cfc92ebd730888cb571fced8757eb6e137ec61b655aba6deb63eb94d" + } +} diff --git a/site/pages/index.md b/site/pages/index.md new file mode 100644 index 00000000..0f714217 --- /dev/null +++ b/site/pages/index.md @@ -0,0 +1,32 @@ +# Morphir Python + +Write a small domain model in Python, compile it to Morphir IR, and generate +Python from that model. Use the library directly from your application or run +the same compiler through the CLI. + +## Start with a model + +Follow the [CLI quickstart](../../docs/quickstart.md) for a complete +compile, inspect, generate and recompile workflow. The +[Python API guide](../../docs/python-compiler.md) performs compilation in memory +and returns typed results and diagnostics. + +## Work with existing IR + +The dependency-free [library](../../docs/library.md) reads and writes Morphir IR, +migrates supported formats, and supplies the Morphir SDK. +[Installation](../../docs/installation.md) explains which package you need. + +## Connect an extension + +The Python compiler also ships as a WASM extension. Start with the +[compatibility table](../../docs/compatibility.md), then follow the guide for +[finos/morphir](../../docs/python-extension.md) or the +[MoonBit-based CLI](../../docs/moonbit.md). Each guide states what its qualified +version can do. + +## Help improve the guides + +These examples are checked against installed packages. The +[contributor guide](../../docs/site.md) explains how to run those checks and +preview this site with Telescope. diff --git a/site/site.json b/site/site.json new file mode 100644 index 00000000..05919f14 --- /dev/null +++ b/site/site.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "title": "Morphir Python", + "description": "Model in Python. Exchange Morphir IR. Generate Python.", + "base_path": "/morphir-python/", + "site_url": "https://finos.github.io/morphir-python/", + "repo_url": "https://github.com/finos/morphir-python", + "repo_ref": "main", + "packages": [], + "api": [], + "source_roots": [ + "docs", + "site/pages" + ], + "nav": [ + { + "group": "Start", + "items": [ + "installation.md", + "quickstart.md" + ] + }, + { + "group": "Python API", + "items": [ + "python-compiler.md", + "library.md", + "api-reference.md" + ] + }, + { + "group": "CLI", + "items": [ + "cli-reference.md" + ] + }, + { + "group": "Connect to Morphir", + "items": [ + "compatibility.md", + "python-extension.md", + "moonbit.md" + ] + }, + { + "group": "Contribute", + "items": [ + "site.md", + "site-foundation.md" + ] + } + ], + "static": [ + { + "from": "site/downloads", + "to": "assets/downloads" + } + ] +} diff --git a/site/toolchain.json b/site/toolchain.json new file mode 100644 index 00000000..e616df5e --- /dev/null +++ b/site/toolchain.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.1", + "platforms": { + "darwin-arm64": { + "url": "https://github.com/moonrockz/telescope/releases/download/v0.2.1/telescope-0.2.1-macos-arm64.tar.gz", + "sha256": "9bac662547f7e52867ad57ef914b9518161952337392e5ed8083623a1117ba4e" + }, + "linux-x64": { + "url": "https://github.com/moonrockz/telescope/releases/download/v0.2.1/telescope-0.2.1-linux-x64.tar.gz", + "sha256": "452fda1f4300f65d2b56e80a526fdc48b4ee027ec0c02f44464566b387cde145" + } + } +} diff --git a/tests/tools/test_python_docs.py b/tests/tools/test_python_docs.py index ce629c3b..82af670c 100644 --- a/tests/tools/test_python_docs.py +++ b/tests/tools/test_python_docs.py @@ -1,42 +1,34 @@ -"""The Python compiler guide's examples run against the public API.""" +"""Published learning examples run against installed APIs and stay type checked.""" from __future__ import annotations +import json import re import subprocess import sys from pathlib import Path -GUIDE = Path(__file__).resolve().parents[2] / "docs" / "python-compiler.md" +from scripts.docs_examples import MARKER, ROOT, run_examples, synchronize -def python_blocks() -> list[str]: - return re.findall(r"```python\n(.*?)```", GUIDE.read_text(), flags=re.DOTALL) +def test_published_snippets_and_references_match_their_sources() -> None: + assert synchronize() == [] + for page in (ROOT / "docs").glob("*.md"): + outside_markers = MARKER.sub("", page.read_text()) + assert not re.search(r"```python\n", outside_markers), page -def test_the_guide_has_an_api_example() -> None: - assert any("compile_sources" in block for block in python_blocks()) - - -def test_api_examples_run_in_a_fresh_interpreter(tmp_path: Path) -> None: - for block in python_blocks(): - completed = subprocess.run( - [sys.executable, "-I", "-c", block], - capture_output=True, - text=True, - cwd=tmp_path, - check=False, - ) - assert completed.returncode == 0, completed.stderr - assert completed.stdout.split() == ["acme/models.py", "acme/rules.py"] - assert list(tmp_path.iterdir()) == [] +def test_named_examples_and_cli_recipe_run_outside_the_checkout() -> None: + result = run_examples(Path(sys.executable)) + assert result["state"] == "succeeded" + assert "examples/cli.sh" in result["examples"] def test_api_examples_pass_both_type_checkers(tmp_path: Path) -> None: - root = Path(__file__).resolve().parents[2] - for index, block in enumerate(python_blocks()): + manifest = json.loads((ROOT / "examples/manifest.json").read_text()) + for index, item in enumerate(manifest["examples"]): example = tmp_path / f"example_{index}.py" - example.write_text(block) + example.write_text((ROOT / item["source"]).read_text()) for command in ( [ sys.executable, @@ -56,6 +48,6 @@ def test_api_examples_pass_both_type_checkers(tmp_path: Path) -> None: ], ): completed = subprocess.run( - command, capture_output=True, text=True, cwd=root, check=False + command, capture_output=True, text=True, cwd=ROOT ) assert completed.returncode == 0, completed.stdout + completed.stderr diff --git a/tests/unit/test_ci_changes.py b/tests/unit/test_ci_changes.py index 5496af5d..a4294a65 100644 --- a/tests/unit/test_ci_changes.py +++ b/tests/unit/test_ci_changes.py @@ -13,20 +13,29 @@ ("paths", "expected"), [ (["README.md", "docs/templates/x.html", "openspec/changes/a/tasks.md"], set()), - (["docs/python-compiler.md"], {"python"}), - (["packages/morphir-python-native/README.md"], {"python", "native"}), + (["docs/python-compiler.md"], {"docs", "docs-examples"}), + ( + ["packages/morphir-python-native/README.md"], + {"python", "native", "docs", "docs-examples"}, + ), ( ["rust/crates/morphir-python-native/src/lib.rs"], {"python", "native", "rust"}, ), (["rust/crates/morphir-python-binding/src/lib.rs"], ALL), (["rust/Cargo.lock"], ALL), - (["packages/morphir/src/morphir/ir/json.py"], {"python", "native"}), - (["uv.lock"], {"python", "native"}), - (["rust/extension.toml"], {"bundle"}), - (["tests/fixtures/python/loans/acme/rules.py"], {"python", "native", "bundle"}), - (["scripts/extension_bundle.py"], {"python", "bundle"}), - ([".gitignore"], {"python", "native"}), + ( + ["packages/morphir/src/morphir/ir/json.py"], + {"python", "native", "docs", "docs-examples"}, + ), + (["uv.lock"], {"python", "native", "docs", "docs-examples"}), + (["rust/extension.toml"], {"bundle", "docs", "docs-examples", "docs-hosts"}), + ( + ["tests/fixtures/python/loans/acme/rules.py"], + {"python", "native", "bundle", "docs", "docs-hosts"}, + ), + (["scripts/extension_bundle.py"], {"python", "bundle", "docs", "docs-hosts"}), + ([".gitignore"], {"python", "native", "docs", "docs-examples"}), ([".github/workflows/ci.yml"], ALL), ([".config/mise/tasks/test"], ALL), (["something/new.txt"], ALL), @@ -58,6 +67,27 @@ def test_full_runs_select_everything() -> None: assert all(c.select([], full=True).jobs().values()) +def test_site_prose_runs_site_checks_without_compiler_builds() -> None: + selection = c.select(["docs/site-foundation.md"]) + assert selection.jobs()["docs"] + assert not selection.jobs()["native-wheel"] + assert not selection.jobs()["rust"] + assert "docs-examples" not in selection.categories + + +def test_examples_select_installed_checks_without_rebuilding_the_native_wheel() -> None: + selection = c.select(["examples/compiler_api.py"]) + assert "docs-examples" in selection.categories + assert selection.jobs()["docs"] + assert not selection.jobs()["native-wheel"] + + +def test_host_recipe_changes_require_fresh_host_evidence() -> None: + selection = c.select(["examples/extension.sh"]) + assert "docs-hosts" in selection.categories + assert selection.jobs()["docs"] + + def needs(**results: str) -> dict[str, dict[str, str]]: return {"changes": {"result": "success"}} | { job.replace("_", "-"): {"result": result} for job, result in results.items() @@ -67,8 +97,9 @@ def needs(**results: str) -> dict[str, dict[str, str]]: def test_aggregate_passes_selected_success_and_unselected_skip() -> None: selected = c.select(["rust/extension.toml"]).jobs() results = needs(**{job.replace("-", "_"): "skipped" for job in selected}) - results["bundle"] = {"result": "success"} - results["native-wheel"] = {"result": "success"} + for job, required in selected.items(): + if required: + results[job] = {"result": "success"} assert c.aggregate(selected, results) == [] diff --git a/tests/unit/test_docs_examples.py b/tests/unit/test_docs_examples.py new file mode 100644 index 00000000..f707f774 --- /dev/null +++ b/tests/unit/test_docs_examples.py @@ -0,0 +1,24 @@ +"""Published snippets must agree with the examples users actually execute.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pathlib import Path + +from scripts.docs_examples import synchronize + + +def test_snippet_drift_fails_check_without_overwriting_the_edit(tmp_path: Path) -> None: + (tmp_path / "examples").mkdir() + (tmp_path / "examples/demo.py").write_text('print("checked")\n') + (tmp_path / "docs").mkdir() + guide = tmp_path / "docs/guide.md" + original = ( + "\n```python\nwrong\n```\n\n" + ) + guide.write_text(original) + assert synchronize(tmp_path, write=False) == ["docs/guide.md"] + assert guide.read_text() == original + assert synchronize(tmp_path, write=True) == ["docs/guide.md"] + assert 'print("checked")' in guide.read_text() + assert synchronize(tmp_path, write=False) == [] diff --git a/tests/unit/test_docs_hosts.py b/tests/unit/test_docs_hosts.py new file mode 100644 index 00000000..d7322e5e --- /dev/null +++ b/tests/unit/test_docs_hosts.py @@ -0,0 +1,77 @@ +"""Compatibility receipts cannot silently survive changed recipes.""" + +import json +from typing import TYPE_CHECKING + +import pytest + +from scripts.docs_hosts import INPUTS, input_hashes, verify_evidence + +if TYPE_CHECKING: + from pathlib import Path + + +def test_edited_recipe_invalidates_previously_successful_evidence( + tmp_path: Path, +) -> None: + for name in INPUTS: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("original") + report = { + "state": "succeeded", + "inputs": input_hashes(tmp_path), + "rust_recipe": {"recipe": "examples/extension.sh", "exit_code": 0}, + "moonbit_refusals": { + v: {"error": "extension.unsupported_model_envelope"} for v in ("3", "4") + }, + } + verify_evidence(json.loads(json.dumps(report)), tmp_path) + (tmp_path / "examples/extension.sh").write_text("changed command") + with pytest.raises(ValueError, match="stale"): + verify_evidence(report, tmp_path) + + +def test_missing_successful_host_result_is_not_reusable(tmp_path: Path) -> None: + for name in INPUTS: + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("original") + with pytest.raises(ValueError, match="lacks"): + verify_evidence( + {"state": "succeeded", "inputs": input_hashes(tmp_path)}, tmp_path + ) + + +def test_released_receipt_can_qualify_a_version_other_than_the_current_crate( + tmp_path: Path, +) -> None: + from scripts import extension_bundle + + version = "1.2.3" + artifact = f"morphir-python-binding-{version}.wasm" + digest = extension_bundle.sha256(b"fixture") + (tmp_path / artifact).write_bytes(b"fixture") + (tmp_path / f"{artifact}.sha256").write_text(f"{digest} {artifact}\n") + descriptor = { + "extensionId": "morphir-python", + "version": version, + "artifacts": [ + { + "sha256": digest, + "claims": { + "extension": {"id": "morphir-python", "version": version}, + "capabilities": { + "frontend": { + "compile": True, + "multiDocument": True, + "irVersions": ["3", "4"], + }, + "backend": {"generate": True, "targets": ["python"]}, + }, + }, + } + ], + } + (tmp_path / "release.json").write_text(json.dumps(descriptor)) + assert extension_bundle.receipt(tmp_path, version=version)["version"] == version diff --git a/tests/unit/test_site_release.py b/tests/unit/test_site_release.py new file mode 100644 index 00000000..6c3fb82e --- /dev/null +++ b/tests/unit/test_site_release.py @@ -0,0 +1,69 @@ +"""Publication refuses output that differs from the checked artifact.""" + +import json +from typing import TYPE_CHECKING + +import pytest + +from scripts.site_release import producer, tree_hashes, verify + +if TYPE_CHECKING: + from pathlib import Path + + +def test_modified_page_blocks_promotion(tmp_path: Path) -> None: + page = tmp_path / "index.html" + page.write_text("Checked") + manifest = { + "state": "succeeded", + "source": "abc", + "eligible_for_deploy": True, + "files": tree_hashes(tmp_path), + } + (tmp_path / "site-evidence.json").write_text(json.dumps(manifest)) + verify(tmp_path, "abc", deploy=True) + page.write_text("Unchecked replacement") + with pytest.raises(ValueError, match="artifact"): + verify(tmp_path, "abc", deploy=True) + + +def test_preview_or_wrong_candidate_cannot_be_promoted(tmp_path: Path) -> None: + (tmp_path / "index.html").write_text("Preview") + manifest = { + "state": "succeeded", + "source": "abc", + "eligible_for_deploy": False, + "files": tree_hashes(tmp_path), + } + (tmp_path / "site-evidence.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="candidate"): + verify(tmp_path, "other", deploy=False) + with pytest.raises(ValueError, match="evidence"): + verify(tmp_path, "abc", deploy=True) + + +@pytest.mark.parametrize( + "changed", + [ + {"event": "pull_request"}, + {"head_branch": "feature"}, + {"head_repository": {"full_name": "someone/fork"}}, + {"conclusion": "cancelled"}, + {"path": ".github/workflows/unrelated.yml"}, + ], +) +def test_only_a_successful_main_ci_producer_is_trusted( + changed: dict[str, object], +) -> None: + run = { + "id": 123, + "head_sha": "a" * 40, + "event": "push", + "head_branch": "main", + "head_repository": {"full_name": "finos/morphir-python"}, + "conclusion": "success", + "path": ".github/workflows/ci.yml", + } + assert producer(run, "finos/morphir-python") == ("a" * 40, 123) + with pytest.raises(ValueError, match="producer"): + producer(run | changed, "finos/morphir-python") diff --git a/tests/unit/test_site_tool.py b/tests/unit/test_site_tool.py new file mode 100644 index 00000000..dbf11768 --- /dev/null +++ b/tests/unit/test_site_tool.py @@ -0,0 +1,65 @@ +"""The site tool installs only verified release archives.""" + +import os +import subprocess +from typing import TYPE_CHECKING + +import pytest + +from scripts.site_tool import SiteError, install_archive, setup + +if TYPE_CHECKING: + from pathlib import Path + + +def test_bad_download_preserves_the_previous_installation(tmp_path: Path) -> None: + archive = tmp_path / "download.tar.gz" + archive.write_bytes(b"corrupt download") + installed = tmp_path / "installed" + installed.mkdir() + (installed / "telescope").write_bytes(b"previous installation") + with pytest.raises(SiteError, match="SHA-256"): + install_archive(archive, installed, "0" * 64, "0.2.1") + assert (installed / "telescope").read_bytes() == b"previous installation" + + +@pytest.mark.skipif( + not os.environ.get("MORPHIR_SITE_TEST"), reason="explicit site qualification" +) +@pytest.mark.parametrize("base", ["/", "/morphir-python/"]) +def test_broken_fragment_preserves_checked_output(tmp_path: Path, base: str) -> None: + import json + + binary = setup() + (tmp_path / "docs/templates").mkdir(parents=True) + (tmp_path / "docs/templates/private.md").write_text("# Privatetemplatetoken\n") + page = tmp_path / "docs/guide.md" + page.write_text("# Guide\n\n## Topic\n\n[Go](#topic)\n") + (tmp_path / "site").mkdir() + (tmp_path / "site/site.json").write_text( + json.dumps( + { + "schema_version": 1, + "title": "Fixture", + "description": "Fixture", + "base_path": base, + "site_url": "https://example.test" + base, + "repo_url": "https://github.com/example/repo", + "repo_ref": "main", + "packages": [], + "source_roots": ["docs"], + } + ) + ) + command = [str(binary), "build", "--config", "site/site.json", "--out", "output"] + subprocess.run(command, cwd=tmp_path, check=True, capture_output=True) + original = (tmp_path / "output/docs/guide/index.html").read_bytes() + assert ( + "Privatetemplatetoken" + not in (tmp_path / "output/search-index.json").read_text() + ) + page.write_text("# Guide\n\n[Missing](#absent)\n") + failed = subprocess.run(command, cwd=tmp_path, capture_output=True, text=True) + assert failed.returncode == 1 + assert "absent" in failed.stderr + failed.stdout + assert (tmp_path / "output/docs/guide/index.html").read_bytes() == original