Versioning & Release Procedure

.github/workflows/test_py_sdist.yml

@@ -9,18 +9,51 @@ on:
   pull_request:
     branches:
       - main
+  release:
+    # Have this run on release so that it produces a Python source distribution
+    # whose version is correctly set to the release's vX.Y.Z tag.
+    branches: [main]
+    types: [published]
 
 jobs:
+  #####----- BUILD SOURCE DISTRIBUTION
+  # Build a single source distribution that will be used for testing across all
+  # different test setups.  This distribution is stored as an artifact so that
+  # it can be manually uploaded directly to PyPI as the official (and well
+  # tested) release.
+  build: 
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+      - name: Setup base Python environment
+        run: $CLONE_PATH/.github/workflows/setup_base_python.sh ${{ runner.os }}
+      - name: Build src dist
+        run: |
+          pushd $PKG_ROOT
+          python -m build --sdist
+          ls -lart ./dist
+          popd
+      - name: Archive distributions
+        uses: actions/upload-artifact@v7
+        with:
+          name: OpenBT-distribution
+          path: |
+            ${{ env.PKG_ROOT }}/dist/openbt-*.tar.gz
+
   #####----- FULL TESTING WITHOUT COVERAGE
-  # Prefer full end-to-end test of local wheel built on demand rather than
-  # testing in local clone alone.
-  #
-  # This also confirms that the test script is functional.
+  # Prefer full end-to-end test of installation built from source distribution
+  # rather than testing in local clone alone.  In particular we test this in the
+  # similar way to how a user might install/test it.
   #
   # Since this is the main test action with many different test setups, we
   # install different MPI implementations via OS package managers.  The devmode
   # action confirms that users can install MPI implementations using PyPI/pip.
   test_sdist:
+    needs: [build]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -75,12 +108,22 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Setup Python dependencies
       run: $CLONE_PATH/.github/workflows/setup_base_python.sh ${{ runner.os }}
+    - uses: actions/download-artifact@v8
+      with:
+        name: OpenBT-distribution
+        path: ${{ env.PKG_ROOT }}/dist
 
     ##-- Run full test suite
     - name: Run full OpenBT test suite
       run: |
-        pushd $PKG_ROOT
-        python -m build --sdist
-        python -m pip install -v dist/openbt-*.tar.gz
-        popd
-        $CLONE_PATH/tools/test_python_installation.py
+        distribution=$PKG_ROOT/dist/openbt-*.tar.gz
+        ls -la $distribution
+        venv=$CLONE_PATH/TestSrc
+        which python
+        python -m venv $venv
+        . $venv/bin/activate
+        which python
+        python -m pip install --upgrade pip
+        python -m pip install -v $distribution
+        python -m pip list
+        python -c "import openbt ; print(openbt.__version__) ; exit(not openbt.test())"

.gitignore

@@ -33,6 +33,7 @@ openbt_pypkg/.coverage_openbt
 openbt_pypkg/coverage.xml
 openbt_pypkg/htmlcov
 openbt_pypkg/src/openbt.egg-info
+openbt_pypkg/src/openbt/_version.py
 
 # Other files
 .DS_Store

docs/index.rst

@@ -55,4 +55,5 @@ This package is being developed as part of |band| framework_.
    git_workflow
    documentation
    versioning
+   release_procedure
    tox_usage

docs/release_procedure.rst

@@ -0,0 +1,158 @@
+Release Procedure
+=================
+
+Pre-release actions
+-------------------
+.. _Python version support: https://devguide.python.org/versions
+.. _Numpy version support: https://numpy.org/neps/nep-0029-deprecation_policy.html#support-table
+.. _Scipy version support: https://docs.scipy.org/doc/scipy/dev/core-dev/index.html#building-binary-installers
+.. _scientific Python support: https://scientific-python.org/specs/spec-0000
+.. _Supported action runner images: https://github.com/actions/runner-images
+.. _specification file: https://github.com/bandframework/OpenBT/blob/main/meson.build
+
+Seed the release process
+
+* Determine the new version identifier ``vX.Y.Z`` needed for the release in
+  accordance with the contents of :numref:`versioning`.
+* Create a new release "super" issue (|ie| a GitHub issue that simply contains
+  a list of links to other GitHub issues).
+
+  * Content could be seeded from previous release's issue
+  * All gatekeepers should converge on set of high-level tasks/goals for release
+  * Create an issue for each approved task and add link to issue in the
+    release "super" issue
+  * As work proceeds, a link to each PR can be placed next to its associated
+    issue's link
+
+* Optionally, create a post-release issue to group together all new issues
+  created during this prerelease/release work that won't be dealt with during
+  the release, but that should be dealt with shortly after the release to
+  maintain inertia and capitalize on the recent work/experience.
+
+  * This should be maintained up-to-date.  Tasks that ultimately can be put off
+    until later can be moved out of this issue.  Issues that will no longer be
+    dealt with as part of the review can be transferred from the release to
+    post-release issue.
+
+* Notify BAND framework team that a new version will be released to determine
+  what steps will need to be made, if any, within the framework once the release
+  is finalized.
+
+Once all tasks have been executed
+
+* Check if a new version of the Eigen package is available through the Meson
+  build system's wrapdb facility and assess if the new version should be adopted
+  and tested.
+* Review and update all metadata in ``setup.py``
+* Review all external dependencies and their stated versions to see if they need
+  updating and address on a feature branch
+
+  * `Python version support`_
+  * `Numpy version support`_
+  * `Scipy version support`_
+  * `scientific Python support`_
+  * Confirm that all version information specified in ``setup.py``,
+    ``pyproject.toml``, and ``tox.ini`` are consistent
+
+* Modernize all repository actions in accord with changes to supported versions
+  and updated GH action infrastructure
+
+  * `Supported action runner images`_
+  * Are there test setups that are presently excluded and that should be added
+    back in?
+
+* Review README including badges for necessary cleaning/updates
+
+  * New references?
+  * Update citation?
+  * Check links
+
+* Review all documentation associated with the repository including examples to
+  determine if any updates still need to be made and address on a feature branch
+
+  * Sphinx/RTD User and Developer Guides
+
+* Confirm continued adherence to all binding requirements (|eg| BAND SDK)
+* Set the version of the C++ command line tools in their Meson build system
+  `specification file`_ to the correct version identifier determined earlier
+
+Release actions
+---------------
+When a particular commit on ``main`` is to be deemed a release,
+
+#. Confirm that all actions ran successfully on the proposed commit
+#. Perform any review of the action logs deemed necessary based on the changes
+   included in the release
+#. Perform any review of the artifacts created by the actions deemed necessary
+   based on the changes included in the release
+#. Tag the release commit with the name ``vX.Y.Z`` and push.  This will trigger
+   the ``test_py_sdist`` GitHub action, which builds the source distribution
+   with the correct version identifier and tests it.
+#. Confirm that the action passed with no errors or warnings.  Review the
+   action's log.
+#. Create a **draft** release with the correct tag ``vX.Y.Z`` and indicate if
+   the release included changes to the C++ command line tools, the Python
+   package, or both.
+#. Carry out all necessary checks for the different software products (see
+   below).
+#. Change the state of the release to **publish**.
+
+Command line tools
+^^^^^^^^^^^^^^^^^^
+#. Gatekeepers to follow installation guide to install and test the command line
+   tools.  This should include a review of the build logs and confirming correct
+   logging of the new release version identifier.
+
+Python package
+^^^^^^^^^^^^^^
+.. _twine instructions: https://twine.readthedocs.io/en/stable/index.html#using-twine
+
+.. note::
+    Since PyPI does not permit the uploading of a revised version to overwrite a
+    previous upload of that same version, these steps should likely be done only
+    **after** having carried out the sanity checks for all other software
+    products.
+
+If the changes in the release do **not** include changes to the C++ command line
+tools or the Python package, then no actions are needed.  In particular, we do
+**not** upload to PyPI the source distributions created with this new versioned
+release.
+
+Otherwise,
+
+#. Download the source distribution artifact from the ``test_py_sdist`` action
+   and confirm that filename contains the desired version identifier.
+#. Decompress the source distribution with ``tar xvfz`` and confirm correct,
+   minimal contents including the presence of the LICENSE file.
+
+   * One side effect of the use of ``setuptools-scm`` is that by default it
+     includes in the distribution all files located within ``openbt_pypkg``.
+     However, many files and folders in that space do **not** need to be
+     distributed (|eg| Flake8 configuration files and OpenBT command line tools
+     installed during development in ``src/openbt/bin``).  This is what
+     "minimal" means above.  Files and folders that should not be included in
+     the distribution are specified in ``MANIFEST.in``.
+   * Review the metadata to ensure correct and complete.  This should include
+     the correct specification of the new version identifier.
+
+#. Gatekeepers to follow installation guide to install and test the Python
+   package in clean virtual environments |via| all provided mechanisms (aside
+   from ``pip install``) as well as by installing from the release's source
+   distribution.
+#. Create a clean virtual environment, update |pip|, and install ``twine``.
+#. Use that venv to publish the downloaded source distribution to PyPI following
+   the `twine instructions`_.  Note that to upload the distribution, you will
+   need to know either your username and password or have an API token that you
+   can generate on the PyPI website under your personal settings.
+#. Review the package's webpage on PyPI.
+#. In a clean virtual environment, follow the installation guide for installing
+   from |pip| and to test the installation
+
+Post-release actions
+--------------------
+* Carry out all necessary tasks to integrate the release within the BAND
+  framework.
+* Update this document based on lessons learned.
+* Review all open issues and pull requests.  Comment and close where possible.
+  Clean-up the post-release issue in preparation for carrying out the work that
+  it requests.

docs/versioning.rst

@@ -1,13 +1,59 @@
+.. _versioning:
+
 Versioning
 ==========
+This repository contains several different, but related software products.  To
+reduce the complexity of managing software versions and to simplify the release
+process, a single version identifier will be assigned to the entire repository
+and its contents rather than designating, for example, a separate version
+identifier to the C++ command line tools, the Python package, and the R package.
+
+Therefore, for this scheme a release might include changes to only one component
+of the repository (|eg| the Python package), but the associated change in the
+version identifier would be applied to all components in the repository.  This
+would include the unaltered R package, which has no direct relationship with the
+Python package.
+
+For the Python package, there would be no need to upload to PyPI a "new" version
+of the package when neither the C++ command line tools nor the Python package
+were altered as part of the changes.  While users can access and use such
+"repeat" versions of the package through a local clone, the majority of users,
+who we assume will install directly from PyPI with ``pip install``, will not be
+exposed to them. They will potentially see, however, skipped versions in the
+package's release history on PyPI.
+
+Versioning rules
+----------------
+.. _SemVer: https://semver.org
+
+* Only commits on the ``main`` branch can be considered for release
+* Version identifiers are strings that adhere to semantic versioning (SemVer_),
+  and the version increment made will be made based on the most significant
+  change or addition made across all software products in the repository
+* Release commits will be tagged using ``vX.Y.Z`` in accordance with
+  ``setuptools-scm`` requirements to allow for automatic versioning of the
+  Python package
+* Each release should be made publicly through the repository's release
+  facilities as offered by GitHub's web interface
+
+C++ command line tools
+----------------------
+.. _specification file: https://github.com/bandframework/OpenBT/blob/main/meson.build
+
+The version of the C++ command line tools is specified in the tools' Meson build
+system `specification file`_.  If a change is made in the tools or its build
+system that requires a new version, it is a natural consequence that all derived
+software products (|eg| the Python package) have their version number altered in
+a consistent way.
+
+Based on our versioning scheme, the tools' version will be updated even if a
+change is made only in one or more of the derived software products.
 
-.. todo::
-    A versioning scheme needs to be decided on and recorded here.  The following
-    are comments in ``meson.build`` that might help seed a conversation about
-    this.  "The version provided here is the version of the C++ library and
-    command line tools.  The Python package maintains a separate version.  If
-    the C++ code is altered, then the version number here should be manually
-    altered and the Python package's version number altered in a similar way
-    since changes made to the C++ code can affect the behavior of the Python
-    code.  If, however, only the Python code is changed, only the version of the
-    Python package should be altered."
+Python package
+--------------
+We prefer to adhere to the standard versioning practices of the Python
+community, which drives the versioning scheme of the whole repository. The
+version of the Python package is managed automatically by the configuration of
+``setuptools-scm`` in ``pyproject.toml``.  The version identifier for the source
+distribution constructed from a tagged commit will automatically be set to the
+tag's name.  Only these distributions should be distributed officially.

meson.build

@@ -1,16 +1,12 @@
 # ----- SPECIFY PROJECT
 # eigen dependency requires at least C++14
 #
-# The version provided here is the version of the C++ library and command line
-# tools.  The Python package maintains a separate version.  If the C++ code is
-# altered, then the version number here should be manually altered and the
-# Python package's version number altered in a similar way since changes made to
-# the C++ code can affect the behavior of the Python code.  If, however, only
-# the Python code is changed, only the version of the Python package should be
-# altered.
+# The version of the C++ command line tools is the X.Y.Z semantic version set
+# into the project below.  For all releases, it should be set to the tag
+# assigned to the release's associated commit.
 project('OpenBT', 'cpp',
         meson_version: '>=1.6.0',
-        version: '1.1.0',
+        version: '1.2.0',
         license: 'MIT',
         license_files: 'LICENSE',
         default_options: [

openbt_pypkg/MANIFEST.in

@@ -1,7 +1,14 @@
 include LICENSE VERSION
+exclude .flake8 .coveragerc
+exclude tox.ini
+prune src/openbt/bin
+prune src/openbt/lib
+prune src/openbt/include
 include cpp/LICENSE
 include cpp/meson.build
 include cpp/meson.options
 include cpp/includes/*.h
 include cpp/src/*.cpp
 include cpp/subprojects/*.wrap
+exclude cpp/subprojects/README.md
+exclude cpp/subprojects/wrapdb.json

openbt_pypkg/pyproject.toml

@@ -1,7 +1,22 @@
+# Since the OpenBT package builds the command line tools during installation, we
+# must specify outside of setup.py (i.e., here) the requirements necessary for
+# build and pip to construct an isolated build system virtual environment with
+# tools needed to build both the command line tools and the package.
+#
+# Since setting the version is also part of building distributions, we also
+# include here the version determination mechanism.
+#
+# All other information needed to compose the package itself is in setup.py.
+
 [build-system]
 requires = [
     "setuptools",
     "ninja",
-    "meson>=1.6.0"
+    "meson>=1.6.0",
+    "setuptools_scm[toml]>=6.0"
 ]
 build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = ".."
+write_to = "openbt_pypkg/src/openbt/_version.py"