diff --git a/.github/workflows/test_py_sdist.yml b/.github/workflows/test_py_sdist.yml index 5713c3e..3084cba 100644 --- a/.github/workflows/test_py_sdist.yml +++ b/.github/workflows/test_py_sdist.yml @@ -9,18 +9,51 @@ on: pull_request: branches: - main + release: + # Have this run on release so that it produces a Python source distribution + # whose version is correctly set to the release's vX.Y.Z tag. + branches: [main] + types: [published] jobs: + #####----- BUILD SOURCE DISTRIBUTION + # Build a single source distribution that will be used for testing across all + # different test setups. This distribution is stored as an artifact so that + # it can be manually uploaded directly to PyPI as the official (and well + # tested) release. + build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.14" + - name: Setup base Python environment + run: $CLONE_PATH/.github/workflows/setup_base_python.sh ${{ runner.os }} + - name: Build src dist + run: | + pushd $PKG_ROOT + python -m build --sdist + ls -lart ./dist + popd + - name: Archive distributions + uses: actions/upload-artifact@v7 + with: + name: OpenBT-distribution + path: | + ${{ env.PKG_ROOT }}/dist/openbt-*.tar.gz + #####----- FULL TESTING WITHOUT COVERAGE - # Prefer full end-to-end test of local wheel built on demand rather than - # testing in local clone alone. - # - # This also confirms that the test script is functional. + # Prefer full end-to-end test of installation built from source distribution + # rather than testing in local clone alone. In particular we test this in the + # similar way to how a user might install/test it. # # Since this is the main test action with many different test setups, we # install different MPI implementations via OS package managers. The devmode # action confirms that users can install MPI implementations using PyPI/pip. test_sdist: + needs: [build] runs-on: ${{ matrix.os }} strategy: matrix: @@ -75,12 +108,22 @@ jobs: python-version: ${{ matrix.python-version }} - name: Setup Python dependencies run: $CLONE_PATH/.github/workflows/setup_base_python.sh ${{ runner.os }} + - uses: actions/download-artifact@v8 + with: + name: OpenBT-distribution + path: ${{ env.PKG_ROOT }}/dist ##-- Run full test suite - name: Run full OpenBT test suite run: | - pushd $PKG_ROOT - python -m build --sdist - python -m pip install -v dist/openbt-*.tar.gz - popd - $CLONE_PATH/tools/test_python_installation.py + distribution=$PKG_ROOT/dist/openbt-*.tar.gz + ls -la $distribution + venv=$CLONE_PATH/TestSrc + which python + python -m venv $venv + . $venv/bin/activate + which python + python -m pip install --upgrade pip + python -m pip install -v $distribution + python -m pip list + python -c "import openbt ; print(openbt.__version__) ; exit(not openbt.test())" diff --git a/.gitignore b/.gitignore index 5b98333..11f91bc 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ openbt_pypkg/.coverage_openbt openbt_pypkg/coverage.xml openbt_pypkg/htmlcov openbt_pypkg/src/openbt.egg-info +openbt_pypkg/src/openbt/_version.py # Other files .DS_Store diff --git a/docs/index.rst b/docs/index.rst index 805e61c..1633abf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,4 +55,5 @@ This package is being developed as part of |band| framework_. git_workflow documentation versioning + release_procedure tox_usage diff --git a/docs/release_procedure.rst b/docs/release_procedure.rst new file mode 100644 index 0000000..8adaee2 --- /dev/null +++ b/docs/release_procedure.rst @@ -0,0 +1,158 @@ +Release Procedure +================= + +Pre-release actions +------------------- +.. _Python version support: https://devguide.python.org/versions +.. _Numpy version support: https://numpy.org/neps/nep-0029-deprecation_policy.html#support-table +.. _Scipy version support: https://docs.scipy.org/doc/scipy/dev/core-dev/index.html#building-binary-installers +.. _scientific Python support: https://scientific-python.org/specs/spec-0000 +.. _Supported action runner images: https://github.com/actions/runner-images +.. _specification file: https://github.com/bandframework/OpenBT/blob/main/meson.build + +Seed the release process + +* Determine the new version identifier ``vX.Y.Z`` needed for the release in + accordance with the contents of :numref:`versioning`. +* Create a new release "super" issue (|ie| a GitHub issue that simply contains + a list of links to other GitHub issues). + + * Content could be seeded from previous release's issue + * All gatekeepers should converge on set of high-level tasks/goals for release + * Create an issue for each approved task and add link to issue in the + release "super" issue + * As work proceeds, a link to each PR can be placed next to its associated + issue's link + +* Optionally, create a post-release issue to group together all new issues + created during this prerelease/release work that won't be dealt with during + the release, but that should be dealt with shortly after the release to + maintain inertia and capitalize on the recent work/experience. + + * This should be maintained up-to-date. Tasks that ultimately can be put off + until later can be moved out of this issue. Issues that will no longer be + dealt with as part of the review can be transferred from the release to + post-release issue. + +* Notify BAND framework team that a new version will be released to determine + what steps will need to be made, if any, within the framework once the release + is finalized. + +Once all tasks have been executed + +* Check if a new version of the Eigen package is available through the Meson + build system's wrapdb facility and assess if the new version should be adopted + and tested. +* Review and update all metadata in ``setup.py`` +* Review all external dependencies and their stated versions to see if they need + updating and address on a feature branch + + * `Python version support`_ + * `Numpy version support`_ + * `Scipy version support`_ + * `scientific Python support`_ + * Confirm that all version information specified in ``setup.py``, + ``pyproject.toml``, and ``tox.ini`` are consistent + +* Modernize all repository actions in accord with changes to supported versions + and updated GH action infrastructure + + * `Supported action runner images`_ + * Are there test setups that are presently excluded and that should be added + back in? + +* Review README including badges for necessary cleaning/updates + + * New references? + * Update citation? + * Check links + +* Review all documentation associated with the repository including examples to + determine if any updates still need to be made and address on a feature branch + + * Sphinx/RTD User and Developer Guides + +* Confirm continued adherence to all binding requirements (|eg| BAND SDK) +* Set the version of the C++ command line tools in their Meson build system + `specification file`_ to the correct version identifier determined earlier + +Release actions +--------------- +When a particular commit on ``main`` is to be deemed a release, + +#. Confirm that all actions ran successfully on the proposed commit +#. Perform any review of the action logs deemed necessary based on the changes + included in the release +#. Perform any review of the artifacts created by the actions deemed necessary + based on the changes included in the release +#. Tag the release commit with the name ``vX.Y.Z`` and push. This will trigger + the ``test_py_sdist`` GitHub action, which builds the source distribution + with the correct version identifier and tests it. +#. Confirm that the action passed with no errors or warnings. Review the + action's log. +#. Create a **draft** release with the correct tag ``vX.Y.Z`` and indicate if + the release included changes to the C++ command line tools, the Python + package, or both. +#. Carry out all necessary checks for the different software products (see + below). +#. Change the state of the release to **publish**. + +Command line tools +^^^^^^^^^^^^^^^^^^ +#. Gatekeepers to follow installation guide to install and test the command line + tools. This should include a review of the build logs and confirming correct + logging of the new release version identifier. + +Python package +^^^^^^^^^^^^^^ +.. _twine instructions: https://twine.readthedocs.io/en/stable/index.html#using-twine + +.. note:: + Since PyPI does not permit the uploading of a revised version to overwrite a + previous upload of that same version, these steps should likely be done only + **after** having carried out the sanity checks for all other software + products. + +If the changes in the release do **not** include changes to the C++ command line +tools or the Python package, then no actions are needed. In particular, we do +**not** upload to PyPI the source distributions created with this new versioned +release. + +Otherwise, + +#. Download the source distribution artifact from the ``test_py_sdist`` action + and confirm that filename contains the desired version identifier. +#. Decompress the source distribution with ``tar xvfz`` and confirm correct, + minimal contents including the presence of the LICENSE file. + + * One side effect of the use of ``setuptools-scm`` is that by default it + includes in the distribution all files located within ``openbt_pypkg``. + However, many files and folders in that space do **not** need to be + distributed (|eg| Flake8 configuration files and OpenBT command line tools + installed during development in ``src/openbt/bin``). This is what + "minimal" means above. Files and folders that should not be included in + the distribution are specified in ``MANIFEST.in``. + * Review the metadata to ensure correct and complete. This should include + the correct specification of the new version identifier. + +#. Gatekeepers to follow installation guide to install and test the Python + package in clean virtual environments |via| all provided mechanisms (aside + from ``pip install``) as well as by installing from the release's source + distribution. +#. Create a clean virtual environment, update |pip|, and install ``twine``. +#. Use that venv to publish the downloaded source distribution to PyPI following + the `twine instructions`_. Note that to upload the distribution, you will + need to know either your username and password or have an API token that you + can generate on the PyPI website under your personal settings. +#. Review the package's webpage on PyPI. +#. In a clean virtual environment, follow the installation guide for installing + from |pip| and to test the installation + +Post-release actions +-------------------- +* Carry out all necessary tasks to integrate the release within the BAND + framework. +* Update this document based on lessons learned. +* Review all open issues and pull requests. Comment and close where possible. + Clean-up the post-release issue in preparation for carrying out the work that + it requests. diff --git a/docs/versioning.rst b/docs/versioning.rst index 93fe611..65c4e07 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -1,13 +1,59 @@ +.. _versioning: + Versioning ========== +This repository contains several different, but related software products. To +reduce the complexity of managing software versions and to simplify the release +process, a single version identifier will be assigned to the entire repository +and its contents rather than designating, for example, a separate version +identifier to the C++ command line tools, the Python package, and the R package. + +Therefore, for this scheme a release might include changes to only one component +of the repository (|eg| the Python package), but the associated change in the +version identifier would be applied to all components in the repository. This +would include the unaltered R package, which has no direct relationship with the +Python package. + +For the Python package, there would be no need to upload to PyPI a "new" version +of the package when neither the C++ command line tools nor the Python package +were altered as part of the changes. While users can access and use such +"repeat" versions of the package through a local clone, the majority of users, +who we assume will install directly from PyPI with ``pip install``, will not be +exposed to them. They will potentially see, however, skipped versions in the +package's release history on PyPI. + +Versioning rules +---------------- +.. _SemVer: https://semver.org + +* Only commits on the ``main`` branch can be considered for release +* Version identifiers are strings that adhere to semantic versioning (SemVer_), + and the version increment made will be made based on the most significant + change or addition made across all software products in the repository +* Release commits will be tagged using ``vX.Y.Z`` in accordance with + ``setuptools-scm`` requirements to allow for automatic versioning of the + Python package +* Each release should be made publicly through the repository's release + facilities as offered by GitHub's web interface + +C++ command line tools +---------------------- +.. _specification file: https://github.com/bandframework/OpenBT/blob/main/meson.build + +The version of the C++ command line tools is specified in the tools' Meson build +system `specification file`_. If a change is made in the tools or its build +system that requires a new version, it is a natural consequence that all derived +software products (|eg| the Python package) have their version number altered in +a consistent way. + +Based on our versioning scheme, the tools' version will be updated even if a +change is made only in one or more of the derived software products. -.. todo:: - A versioning scheme needs to be decided on and recorded here. The following - are comments in ``meson.build`` that might help seed a conversation about - this. "The version provided here is the version of the C++ library and - command line tools. The Python package maintains a separate version. If - the C++ code is altered, then the version number here should be manually - altered and the Python package's version number altered in a similar way - since changes made to the C++ code can affect the behavior of the Python - code. If, however, only the Python code is changed, only the version of the - Python package should be altered." +Python package +-------------- +We prefer to adhere to the standard versioning practices of the Python +community, which drives the versioning scheme of the whole repository. The +version of the Python package is managed automatically by the configuration of +``setuptools-scm`` in ``pyproject.toml``. The version identifier for the source +distribution constructed from a tagged commit will automatically be set to the +tag's name. Only these distributions should be distributed officially. diff --git a/meson.build b/meson.build index ff94b94..69f2e3f 100644 --- a/meson.build +++ b/meson.build @@ -1,16 +1,12 @@ # ----- SPECIFY PROJECT # eigen dependency requires at least C++14 # -# The version provided here is the version of the C++ library and command line -# tools. The Python package maintains a separate version. If the C++ code is -# altered, then the version number here should be manually altered and the -# Python package's version number altered in a similar way since changes made to -# the C++ code can affect the behavior of the Python code. If, however, only -# the Python code is changed, only the version of the Python package should be -# altered. +# The version of the C++ command line tools is the X.Y.Z semantic version set +# into the project below. For all releases, it should be set to the tag +# assigned to the release's associated commit. project('OpenBT', 'cpp', meson_version: '>=1.6.0', - version: '1.1.0', + version: '1.2.0', license: 'MIT', license_files: 'LICENSE', default_options: [ diff --git a/openbt_pypkg/MANIFEST.in b/openbt_pypkg/MANIFEST.in index 762761c..9f8c6e7 100644 --- a/openbt_pypkg/MANIFEST.in +++ b/openbt_pypkg/MANIFEST.in @@ -1,7 +1,14 @@ include LICENSE VERSION +exclude .flake8 .coveragerc +exclude tox.ini +prune src/openbt/bin +prune src/openbt/lib +prune src/openbt/include include cpp/LICENSE include cpp/meson.build include cpp/meson.options include cpp/includes/*.h include cpp/src/*.cpp include cpp/subprojects/*.wrap +exclude cpp/subprojects/README.md +exclude cpp/subprojects/wrapdb.json diff --git a/openbt_pypkg/VERSION b/openbt_pypkg/VERSION deleted file mode 100644 index 781dcb0..0000000 --- a/openbt_pypkg/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.1.3 diff --git a/openbt_pypkg/pyproject.toml b/openbt_pypkg/pyproject.toml index 975c9fa..8a4e5d4 100644 --- a/openbt_pypkg/pyproject.toml +++ b/openbt_pypkg/pyproject.toml @@ -1,7 +1,22 @@ +# Since the OpenBT package builds the command line tools during installation, we +# must specify outside of setup.py (i.e., here) the requirements necessary for +# build and pip to construct an isolated build system virtual environment with +# tools needed to build both the command line tools and the package. +# +# Since setting the version is also part of building distributions, we also +# include here the version determination mechanism. +# +# All other information needed to compose the package itself is in setup.py. + [build-system] requires = [ "setuptools", "ninja", - "meson>=1.6.0" + "meson>=1.6.0", + "setuptools_scm[toml]>=6.0" ] build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +root = ".." +write_to = "openbt_pypkg/src/openbt/_version.py" diff --git a/openbt_pypkg/setup.py b/openbt_pypkg/setup.py index 048cedf..66c8172 100644 --- a/openbt_pypkg/setup.py +++ b/openbt_pypkg/setup.py @@ -1,3 +1,12 @@ +# IMPORTANT +# * If necessary, please adjust pyproject.toml and GitHub actions to match +# changes to Python version here. +# * Please make sure that all dependence/version changes made here are reflected +# in tox.ini. +# +# The version is being set in _version.py by setuptools_scm (see +# pyproject.toml). No need to handle version manually here. + import os import sys import shutil @@ -23,7 +32,7 @@ # Package metadata PYTHON_REQUIRES = ">=3.10" -CODE_REQUIRES = ["numpy<=2.4.6", "matplotlib"] +CODE_REQUIRES = ["numpy", "matplotlib"] TEST_REQUIRES = ["pytest", "scipy", "pandas"] INSTALL_REQUIRES = CODE_REQUIRES + TEST_REQUIRES @@ -107,15 +116,8 @@ def readme_md(): return fptr.read() -def version(): - fname = PKG_ROOT.joinpath("VERSION") - with open(fname, "r") as fptr: - return fptr.read().strip() - - setup( name='openbt', - version=version(), author="John Yannotty", author_email="yannotty.1@buckeyemail.osu.edu", maintainer="John Yannotty",