diff --git a/.github/release_drafter.yaml b/.github/release_drafter.yaml deleted file mode 100644 index 412adf1ee..000000000 --- a/.github/release_drafter.yaml +++ /dev/null @@ -1,35 +0,0 @@ -version-resolver: - major: - labels: - - 'breaking' - minor: - labels: - - 'feature' - patch: - labels: - - 'documentation' - - 'enhancement' - - 'bug' -name-template: '$RESOLVED_VERSION' -tag-template: 'v$RESOLVED_VERSION' -categories: - - title: '💥 Breaking Changes' - labels: - - 'breaking' - - title: '🚀 Features' - labels: - - 'feature' - - title: '⚡️ Enhancements' - labels: - - 'enhancement' - - title: '🐛 Bug Fixes' - labels: - - 'bug' - - title: '📝 Documentation' - label: 'documentation' -change-template: '- $TITLE(#$NUMBER)@$AUTHOR' -change-title-escapes: '\<*_&@#' # escape @ and # so decorator/issue refs in PR titles (e.g. `@schema`) don't render as spurious mentions/links in release notes. -template: | - $CHANGES - - **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...v$RESOLVED_VERSION \ No newline at end of file diff --git a/.github/scripts/check_release_tag.sh b/.github/scripts/check_release_tag.sh new file mode 100755 index 000000000..9b1b44065 --- /dev/null +++ b/.github/scripts/check_release_tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Decide whether a release tag can be published. +# +# check_release_tag.sh [] +# +# Fails on a tag that is not vX.Y.Z, on a lightweight tag, and on a tag whose +# commit is not on (default: master). Succeeds without publishing on a +# tag whose message has a "Retroactive:" line: such tags record past releases. +# Writes publish=true or publish=false to $GITHUB_OUTPUT when it is set. +set -euo pipefail + +tag="${1:?usage: check_release_tag.sh []}" +branch="${2:-master}" +output="${GITHUB_OUTPUT:-/dev/null}" + +fail() { + echo "::error::$1" >&2 + shift + for line in "$@"; do echo "$line" >&2; done + exit 1 +} + +[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "$tag is not a release tag (vX.Y.Z)." + +# actions/checkout can store an annotated tag as a lightweight ref, so fetch the +# tag object itself, and the branch the tag must be on. +git fetch --quiet --force origin "refs/tags/$tag:refs/tags/$tag" +git fetch --quiet --no-tags origin "+refs/heads/$branch:refs/remotes/origin/$branch" + +if [[ "$(git cat-file -t "refs/tags/$tag")" != "tag" ]]; then + fail "$tag is a lightweight tag. A release tag is annotated, with the release notes as its message:" \ + " git tag -d $tag" \ + " git tag -a $tag --cleanup=verbatim -F notes.md" \ + " git push --force origin refs/tags/$tag" +fi + +commit="$(git rev-parse "refs/tags/$tag^{commit}")" +if ! git merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then + fail "$tag points to $commit, which is not on $branch." +fi + +if git for-each-ref "refs/tags/$tag" --format='%(contents)' | grep -q '^Retroactive:'; then + echo "::notice::$tag records a past release (its message has a Retroactive: line); nothing to publish." + echo "publish=false" >> "$output" + exit 0 +fi + +echo "$tag is an annotated tag on $branch; publishing." +echo "publish=true" >> "$output" diff --git a/.github/workflows/draft_release.yaml b/.github/workflows/draft_release.yaml deleted file mode 100644 index 790ce6c3c..000000000 --- a/.github/workflows/draft_release.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Manual Draft Release -on: - workflow_dispatch: - inputs: - testpypi: - description: 'Release to TestPyPI then skip following' - default: 'false' - type: choice - options: - - 'true' - - 'false' -jobs: - build-release: - permissions: - # write permission is required to create a github release - contents: write - # write permission is required for autolabeler - # otherwise, read permission is required at least - pull-requests: read - runs-on: ubuntu-latest - steps: - # Drafts your next Release notes as Pull Requests are merged into "master" - - name: Draft release notes - id: create_gh_release - uses: release-drafter/release-drafter@v6 - with: - config-name: release_drafter.yaml - disable-autolabeler: true - name: ${{ github.event.inputs.testpypi == 'true' && 'Test DataJoint Library $RESOLVED_VERSION' || 'DataJoint Library $RESOLVED_VERSION' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e7e6dc2ae..b9e11bd77 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -10,6 +10,8 @@ on: - "**" # every branch - "!gh-pages" # exclude gh-pages branch - "!stage*" # exclude branches beginning with stage + # The release workflow runs this as its gate. + workflow_call: jobs: lint: runs-on: ubuntu-latest diff --git a/.github/workflows/post_draft_release_published.yaml b/.github/workflows/post_draft_release_published.yaml deleted file mode 100644 index 266887664..000000000 --- a/.github/workflows/post_draft_release_published.yaml +++ /dev/null @@ -1,108 +0,0 @@ -name: Post Draft Release Published - -on: - release: - types: - ## pre-release and stable release - #- published - ## stable release only - - released -run-name: Post ${{ github.event.release.name }} - -jobs: - pypi-release: - permissions: - # write permission is required to upload build artifacts as release assets - contents: write - # Use the oldest supported version to build, just in case there are issues - # for our case, this doesn't matter that much, since the build is for 3.x - strategy: - matrix: - include: - - py_ver: "3.10" - runs-on: ubuntu-latest - env: - PY_VER: ${{matrix.py_ver}} - TWINE_USERNAME: ${{secrets.twine_username}} - TWINE_PASSWORD: ${{secrets.twine_password}} - TWINE_TEST_USERNAME: ${{secrets.twine_test_username}} - TWINE_TEST_PASSWORD: ${{secrets.twine_test_password}} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - # Full history + tags so hatch-vcs can derive the version from the - # release tag. The checked-out ref is the tag itself, so the build - # reports exactly X.Y.Z — no version.py bump or follow-up PR needed. - fetch-depth: 0 - - name: Set up Python ${{matrix.py_ver}} - uses: actions/setup-python@v5 - with: - python-version: ${{matrix.py_ver}} - # Merging build and release steps just for the simplicity, - # since datajoint-python doesn't have platform specific dependencies or binaries, - # and the build process is fairly fast, so removed upload/download artifacts - - name: Build package - id: build - run: | - python -m pip install build - python -m build . - echo "DJ_WHEEL_PATH=$(ls dist/datajoint-*.whl)" >> $GITHUB_ENV - echo "DJ_SDIST_PATH=$(ls dist/datajoint-*.tar.gz)" >> $GITHUB_ENV - - name: Publish package - id: publish - env: - RELEASE_NAME: ${{ github.event.release.name }} - run: | - python -m pip install --upgrade twine - if [[ "$RELEASE_NAME" =~ ^Test ]]; then - echo "TEST_PYPI=true" >> $GITHUB_ENV - export TWINE_REPOSITORY="testpypi" - export TWINE_USERNAME=${TWINE_TEST_USERNAME} - export TWINE_PASSWORD=${TWINE_TEST_PASSWORD} - else - echo "TEST_PYPI=false" >> $GITHUB_ENV - export TWINE_REPOSITORY="pypi" - fi - # Upload the artifacts built in the previous step. --skip-existing - # keeps re-runs idempotent (no error if the version already exists). - python -m twine upload --skip-existing dist/* - # Upload package as release assets - - name: Upload pip wheel asset to release - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - with: - upload_url: ${{github.event.release.upload_url}} - asset_path: ${{env.DJ_WHEEL_PATH}} - asset_name: pip-datajoint-${{ github.event.release.tag_name }}.whl - asset_content_type: application/zip - - name: Upload pip sdist asset to release - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - with: - upload_url: ${{github.event.release.upload_url}} - asset_path: ${{env.DJ_SDIST_PATH}} - asset_name: pip-datajoint-${{ github.event.release.tag_name }}.tar.gz - asset_content_type: application/gzip - - name: Post release notification to Slack - if: ${{ env.TEST_PYPI == 'false' }} - uses: slackapi/slack-github-action@v2.0.0 - with: - webhook: ${{ secrets.SLACK_WEBHOOK_URL }} - webhook-type: incoming-webhook - payload: | - { - "text": "*New Release Published!* :tada: \n*Repository:* ${{ github.repository }}\n*Version:* ${{ github.event.release.tag_name }}\n*URL:* ${{ github.event.release.html_url }}", - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "*New Release Published!* :tada:\n*Repository:* ${{ github.repository }}\n*Version:* ${{ github.event.release.tag_name }}\n*URL:* <${{ github.event.release.html_url }}|View Release>" - } - } - ] - } diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 000000000..8f392573c --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,134 @@ +name: Release + +# A release is an annotated tag vX.Y.Z on master whose message is the release +# notes. Pushing it runs the test suite, builds the package at the tag (the +# version comes from the tag through hatch-vcs), publishes it to PyPI, and +# creates the GitHub release with the tag's message as its notes. +# +# To publish an existing tag to TestPyPI instead, run this workflow by hand from +# that tag ("Use workflow from" -> Tags) with testpypi checked. +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + workflow_dispatch: + inputs: + testpypi: + description: "Publish to TestPyPI, without a GitHub release or notification" + type: boolean + default: true + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-latest + outputs: + publish: ${{ steps.check.outputs.publish }} + steps: + - name: Require a tag + if: github.ref_type != 'tag' + run: | + echo "::error::Run this workflow from a release tag (vX.Y.Z), not from a branch." + exit 1 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check the tag + id: check + run: .github/scripts/check_release_tag.sh "$GITHUB_REF_NAME" master + + lint: + needs: check + if: needs.check.outputs.publish == 'true' + uses: ./.github/workflows/lint.yaml + + test: + needs: check + if: needs.check.outputs.publish == 'true' + uses: ./.github/workflows/test.yaml + + publish: + needs: [check, lint, test] + if: needs.check.outputs.publish == 'true' + runs-on: ubuntu-latest + permissions: + # write permission is required to create the GitHub release + contents: write + env: + TAG: ${{ github.ref_name }} + TEST_PYPI: ${{ github.event_name == 'workflow_dispatch' && inputs.testpypi }} + TWINE_USERNAME: ${{ secrets.twine_username }} + TWINE_PASSWORD: ${{ secrets.twine_password }} + TWINE_TEST_USERNAME: ${{ secrets.twine_test_username }} + TWINE_TEST_PASSWORD: ${{ secrets.twine_test_password }} + steps: + - uses: actions/checkout@v4 + with: + # Full history and tags, so hatch-vcs derives the version from the tag. + fetch-depth: 0 + - name: Fetch the tag object + # actions/checkout can store an annotated tag as a lightweight ref. + run: git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG" + - name: Set up Python + uses: actions/setup-python@v5 + with: + # The oldest supported Python; the build is pure Python either way. + python-version: "3.10" + - name: Build + run: | + python -m pip install build + python -m build . + - name: Check the built version + run: | + version="${TAG#v}" + for file in dist/*; do + case "$(basename "$file")" in + "datajoint-$version-"*.whl | "datajoint-$version.tar.gz") ;; + *) echo "::error::$file does not carry version $version"; exit 1 ;; + esac + done + - name: Publish to PyPI + run: | + python -m pip install --upgrade twine + if [[ "$TEST_PYPI" == "true" ]]; then + export TWINE_REPOSITORY="testpypi" + export TWINE_USERNAME="$TWINE_TEST_USERNAME" + export TWINE_PASSWORD="$TWINE_TEST_PASSWORD" + else + export TWINE_REPOSITORY="pypi" + fi + # --skip-existing keeps re-runs idempotent. + python -m twine upload --skip-existing dist/* + - name: Create the GitHub release + if: env.TEST_PYPI != 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + git for-each-ref "refs/tags/$TAG" --format='%(contents)' > "$RUNNER_TEMP/notes.md" + gh release create "$TAG" --verify-tag --title "DataJoint Library ${TAG#v}" \ + --notes-file "$RUNNER_TEMP/notes.md" dist/* + - name: Post release notification to Slack + if: env.TEST_PYPI != 'true' + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: incoming-webhook + payload: | + { + "text": "*New Release Published!* :tada: \n*Repository:* ${{ github.repository }}\n*Version:* ${{ github.ref_name }}\n*URL:* ${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "*New Release Published!* :tada:\n*Repository:* ${{ github.repository }}\n*Version:* ${{ github.ref_name }}\n*URL:* <${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}|View Release>" + } + } + ] + } diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4bd734234..59fc3ba51 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,6 +24,9 @@ on: - "pixi.lock" - ".github/workflows/test.yaml" + # The release workflow runs this as its gate. + workflow_call: + jobs: test: runs-on: ubuntu-latest diff --git a/CHANGELOG-archive.md b/CHANGELOG-archive.md deleted file mode 100644 index 46241c669..000000000 --- a/CHANGELOG-archive.md +++ /dev/null @@ -1,349 +0,0 @@ -# Changelog Archive - -> **This file is archived.** For current release notes, see: -> -> **[GitHub Releases](https://github.com/datajoint/datajoint-python/releases)** -> -> Release notes are now automatically generated from pull request labels and descriptions. -> This file preserves the history of releases through version 0.14.3. - ---- - -## Historical Release Notes - -### 0.14.3 -- Sep 23, 2024 -- Added - `dj.Top` restriction - PR [#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084) -- Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177) -- Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) - PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095) -- Added - Ability to set hidden attributes on a table - PR [#1091](https://github.com/datajoint/datajoint-python/pull/1091) -- Added - Ability to specify a list of keys to populate - PR [#989](https://github.com/datajoint/datajoint-python/pull/989) -- Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057) - PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) -- Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103) - PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) -- Changed - replace `setup.py` with `pyproject.toml` - PR [#1183](https://github.com/datajoint/datajoint-python/pull/1183) -- Changed - disable `add_hidden_timestamp` configuration option by default - PR [#1188](https://github.com/datajoint/datajoint-python/pull/1188) - -### 0.14.2 -- Aug 19, 2024 -- Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142) -- Added - Codespell GitHub Actions workflow -- Added - GitHub Actions workflow to manually release docs -- Changed - Update `datajoint/nginx` to `v0.2.6` -- Changed - Migrate docs from `https://docs.datajoint.org/python` to `https://datajoint.com/docs/core/datajoint-python` -- Fixed - [DevContainer](https://containers.dev/) configuration - PR [#1115](https://github.com/datajoint/datajoint-python/pull/1115) -- Fixed - Updated set_password to work on MySQL 8 - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106) -- Added - Missing tests for set_password - PR [#1106](https://github.com/datajoint/datajoint-python/pull/1106) -- Changed - Returning success count after the .populate() call - PR [#1050](https://github.com/datajoint/datajoint-python/pull/1050) -- Fixed - `Autopopulate.populate` excludes `reserved` jobs in addition to `ignore` and `error` jobs -- Fixed - Issue [#1159](https://github.com/datajoint/datajoint-python/pull/1159) (cascading delete) - PR [#1160](https://github.com/datajoint/datajoint-python/pull/1160) -- Changed - Minimum Python version for Datajoint-Python is now 3.8 PR #1163 -- Fixed - `docker compose` commands in CI [#1164](https://github.com/datajoint/datajoint-python/pull/1164) -- Changed - Default delete behavior now includes masters of part tables - PR [#1158](https://github.com/datajoint/datajoint-python/pull/1158) - -### 0.14.1 -- Jun 02, 2023 -- Fixed - Fix altering a part table that uses the "master" keyword - PR [#991](https://github.com/datajoint/datajoint-python/pull/991) -- Fixed - `.ipynb` output in tutorials is not visible in dark mode ([#1078](https://github.com/datajoint/datajoint-python/issues/1078)) PR [#1080](https://github.com/datajoint/datajoint-python/pull/1080) -- Fixed - preview table font for darkmode PR [#1089](https://github.com/datajoint/datajoint-python/pull/1089) -- Changed - Readme to update links and include example pipeline image -- Changed - Docs to add landing page and update navigation -- Changed - `.data` method to `.stream` in the `get()` method for S3 (external) objects PR [#1085](https://github.com/datajoint/datajoint-python/pull/1085) -- Fixed - Docs to rename `create_virtual_module` to `VirtualModule` -- Added - Skeleton from `datajoint-company/datajoint-docs` repository for docs migration -- Added - Initial `pytest` for `test_connection` - -### 0.14.0 -- Feb 13, 2023 -- Added - `json` data type ([#245](https://github.com/datajoint/datajoint-python/issues/245)) PR [#1051](https://github.com/datajoint/datajoint-python/pull/1051) -- Fixed - Activating a schema requires all tables to exist even if `create_tables=False` PR [#1058](https://github.com/datajoint/datajoint-python/pull/1058) -- Changed - Populate call with `reserve_jobs=True` to exclude `error` and `ignore` keys - PR [#1062](https://github.com/datajoint/datajoint-python/pull/1062) -- Added - Support for inserting data with CSV files - PR [#1067](https://github.com/datajoint/datajoint-python/pull/1067) -- Changed - Switch testing image from `pydev` to `djtest` PR [#1012](https://github.com/datajoint/datajoint-python/pull/1012) -- Added - DevContainer development environment compatible with GH Codespaces PR [1071](https://github.com/datajoint/datajoint-python/pull/1071) -- Fixed - Convert lingering prints by replacing with logs PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Changed - `table.progress()` defaults to no stdout PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Changed - `table.describe()` defaults to no stdout PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Deprecated - `table._update()` PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Deprecated - old-style foreign key syntax PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Deprecated - `dj.migrate_dj011_external_blob_storage_to_dj012()` PR [#1073](https://github.com/datajoint/datajoint-python/pull/1073) -- Added - Method to set job keys to "ignore" status - PR [#1068](https://github.com/datajoint/datajoint-python/pull/1068) - -### 0.13.8 -- Sep 21, 2022 -- Added - New documentation structure based on markdown PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052) -- Fixed - Fix queries with backslashes ([#999](https://github.com/datajoint/datajoint-python/issues/999)) PR [#1052](https://github.com/datajoint/datajoint-python/pull/1052) - -### 0.13.7 -- Jul 13, 2022 -- Fixed - Fix networkx incompatible change by version pinning to 2.6.3 (#1035) PR #1036 -- Added - Support for serializing numpy datetime64 types (#1022) PR #1036 -- Changed - Add traceback to default logging PR #1036 - -### 0.13.6 -- Jun 13, 2022 -- Added - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025 -- Added - Unified package level logger for package (#667) PR #1031 -- Changed - Swap various datajoint messages, warnings, etc. to use the new logger. (#667) PR #1031 -- Fixed - Fix query caching deleting non-datajoint files PR #1027 -- Changed - Minimum Python version for Datajoint-Python is now 3.7 PR #1027 - -### 0.13.5 -- May 19, 2022 -- Changed - Import ABC from collections.abc for Python 3.10 compatibility -- Fixed - Fix multiprocessing value error (#1013) PR #1026 - -### 0.13.4 -- Mar, 28 2022 -- Added - Allow reading blobs produced by legacy 32-bit compiled mYm library for matlab. PR #995 -- Fixed - Add missing `jobs` argument for multiprocessing PR #997 -- Added - Test for multiprocessing PR #1008 -- Fixed - Fix external store key name doesn't allow '-' (#1005) PR #1006 -- Added - Adopted black formatting into code base PR #998 - -### 0.13.3 -- Feb 9, 2022 -- Fixed - Fix error in listing ancestors, descendants with part tables. -- Fixed - Fix Python 3.10 compatibility (#983) PR #972 -- Fixed - Allow renaming non-conforming attributes in proj (#982) PR #972 -- Added - Expose proxy feature for S3 external stores (#961) PR #962 -- Added - implement multiprocessing in populate (#695) PR #704, #969 -- Fixed - Dependencies not properly loaded on populate. (#902) PR #919 -- Fixed - Replace use of numpy aliases of built-in types with built-in type. (#938) PR #939 -- Fixed - Deletes and drops must include the master of each part. (#151, #374) PR #957 -- Fixed - `ExternalTable.delete` should not remove row on error (#953) PR #956 -- Fixed - Fix error handling of remove_object function in `s3.py` (#952) PR #955 -- Fixed - Fix regression issue with `DISTINCT` clause and `GROUP_BY` (#914) PR #963 -- Fixed - Fix sql code generation to comply with sql mode `ONLY_FULL_GROUP_BY` (#916) PR #965 -- Fixed - Fix count for left-joined `QueryExpressions` (#951) PR #966 -- Fixed - Fix assertion error when performing a union into a join (#930) PR #967 -- Changed `~jobs.error_stack` from blob to mediumblob to allow error stacks >64kB in jobs (#984) PR #986 -- Fixed - Fix error when performing a union on multiple tables (#926) PR #964 -- Added - Allow optional keyword arguments for `make()` in `populate()` PR #971 - -### 0.13.2 -- May 7, 2021 -- Changed `setuptools_certificate` dependency to new name `otumat` -- Fixed - Explicit calls to `dj.Connection` throw error due to missing `host_input` (#895) PR #907 -- Fixed - Correct count of deleted items. (#897) PR #912 - -### 0.13.1 -- Apr 16, 2021 -- Added `None` as an alias for `IS NULL` comparison in `dict` restrictions (#824) PR #893 -- Changed - Drop support for MySQL 5.6 since it has reached EOL PR #893 -- Fixed - `schema.list_tables()` is not topologically sorted (#838) PR #893 -- Fixed - Diagram part tables do not show proper class name (#882) PR #893 -- Fixed - Error in complex restrictions (#892) PR #893 -- Fixed - WHERE and GROUP BY classes are dropped on joins with aggregation (#898, #899) PR #893 - -### 0.13.0 -- Mar 24, 2021 -- Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484, #558). PR #754 -- Re-implement cascading deletes for better performance. PR #839 -- Add support for deferred schema activation to allow for greater modularity. (#834) PR #839 -- Add query caching mechanism for offline development (#550) PR #839 -- Add table method `.update1` to update a row in the table with new values (#867) PR #763, #889 -- Python datatypes are now enabled by default in blobs (#761). PR #859 -- Added permissive join and restriction operators `@` and `^` (#785) PR #754 -- Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735 -- Add `dj.key_hash` alias to `dj.hash.key_hash` (#804) PR #862 -- Default enable_python_native_blobs to True -- Bugfix - Regression error on joins with same attribute name (#857) PR #878 -- Bugfix - Error when `fetch1('KEY')` when `dj.config['fetch_format']='frame'` set (#876) PR #880, #878 -- Bugfix - Error when cascading deletes in tables with many, complex keys (#883, #886) PR #839 -- Add deprecation warning for `_update`. PR #889 -- Add `purge_query_cache` utility. PR #889 -- Add tests for query caching and permissive join and restriction. PR #889 -- Drop support for Python 3.5 (#829) PR #861 - -### 0.12.9 -- Mar 12, 2021 -- Fix bug with fetch1 with `dj.config['fetch_format']="frame"`. (#876) PR #880 - -### 0.12.8 -- Jan 12, 2021 -- table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833 -- Load dependencies before querying dependencies. (#179) PR #833 -- Fix display of part tables in `schema.save`. (#821) PR #833 -- Add `schema.list_tables`. (#838) PR #844 -- Fix minio new version regression. PR #847 -- Add more S3 logging for debugging. (#831) PR #832 -- Convert testing framework from TravisCI to GitHub Actions (#841) PR #840 - -### 0.12.7 -- Oct 27, 2020 -- Fix case sensitivity issues to adapt to MySQL 8+. PR #819 -- Fix pymysql regression bug (#814) PR #816 -- Adapted attribute types now have dtype=object in all recarray results. PR #811 - -### 0.12.6 -- May 15, 2020 -- Add `order_by` to `dj.kill` (#668, #779) PR #775, #783 -- Add explicit S3 bucket and file storage location existence checks (#748) PR #781 -- Modify `_update` to allow nullable updates for strings/date (#664) PR #760 -- Avoid logging events on auxiliary tables (#737) PR #753 -- Add `kill_quick` and expand display to include host (#740) PR #741 -- Bugfix - pandas insert fails due to additional `index` field (#666) PR #776 -- Bugfix - `delete_external_files=True` does not remove from S3 (#686) PR #781 -- Bugfix - pandas fetch throws error when `fetch_format='frame'` PR #774 - -### 0.12.5 -- Feb 24, 2020 -- Rename module `dj.schema` into `dj.schemas`. `dj.schema` remains an alias for class `dj.Schema`. (#731) PR #732 -- `dj.create_virtual_module` is now called `dj.VirtualModule` (#731) PR #732 -- Bugfix - SSL `KeyError` on failed connection (#716) PR #725 -- Bugfix - Unable to run unit tests using nosetests (#723) PR #724 -- Bugfix - `suppress_errors` does not suppress loss of connection error (#720) PR #721 - -### 0.12.4 -- Jan 14, 2020 -- Support for simple scalar datatypes in blobs (#690) PR #709 -- Add support for the `serial` data type in declarations: alias for `bigint unsigned auto_increment` PR #713 -- Improve the log table to avoid primary key collisions PR #713 -- Improve documentation in README PR #713 - -### 0.12.3 -- Nov 22, 2019 -- Bugfix - networkx 2.4 causes error in diagrams (#675) PR #705 -- Bugfix - include table definition in doc string and help (#698, #699) PR #706 -- Bugfix - job reservation fails when native python datatype support is disabled (#701) PR #702 - -### 0.12.2 -- Nov 11, 2019 -- Bugfix - Convoluted error thrown if there is a reference to a non-existent table attribute (#691) PR #696 -- Bugfix - Insert into external does not trim leading slash if defined in `dj.config['stores']['']['location']` (#692) PR #693 - -### 0.12.1 -- Nov 2, 2019 -- Bugfix - AttributeAdapter converts into a string (#684) PR #688 - -### 0.12.0 -- Oct 31, 2019 -- Dropped support for Python 3.4 -- Support secure connections with TLS (aka SSL) PR #620 -- Convert numpy array from python object to appropriate data type if all elements are of the same type (#587) PR #608 -- Remove expression requirement to have additional attributes (#604) PR #604 -- Support for filepath datatype (#481) PR #603, #659 -- Support file attachment datatype (#480, #592, #637) PR #659 -- Fetch return a dict array when specifying `as_dict=True` for specified attributes. (#595) PR #593 -- Support of ellipsis in `proj`: `query_expression.proj(.., '-movie')` (#499) PR #578 -- Expand support of blob serialization (#572, #520, #427, #392, #244, #594) PR #577 -- Support for alter (#110) PR #573 -- Support for `conda install datajoint` via `conda-forge` channel (#293) -- `dj.conn()` accepts a `port` keyword argument (#563) PR #571 -- Support for UUID datatype (#562) PR #567 -- `query_expr.fetch("KEY", as_dict=False)` returns results as `np.recarray`(#414) PR #574 -- `dj.ERD` is now called `dj.Diagram` (#255, #546) PR #565 -- `dj.Diagram` underlines "distinguished" classes (#378) PR #557 -- Accept alias for supported MySQL datatypes (#544) PR #545 -- Support for pandas in `fetch` (#459, #537) PR #534 -- Support for ordering by "KEY" in `fetch` (#541) PR #534 -- Add config to enable python native blobs PR #672, #676 -- Add secure option for external storage (#663) PR #674, #676 -- Add blob migration utility from DJ011 to DJ012 PR #673 -- Improved external storage - a migration script needed from version 0.11 (#467, #475, #480, #497) PR #532 -- Increase default display rows (#523) PR #526 -- Bugfixes (#521, #205, #279, #477, #570, #581, #597, #596, #618, #633, #643, #644, #647, #648, #650, #656) -- Minor improvements (#538) - -### 0.11.3 -- Jul 26, 2019 -- Fix incompatibility with pyparsing 2.4.1 (#629) PR #631 - -### 0.11.2 -- Jul 25, 2019 -- Fix #628 - incompatibility with pyparsing 2.4.1 - -### 0.11.1 -- Nov 15, 2018 -- Fix ordering of attributes in proj (#483, #516) -- Prohibit direct insert into auto-populated tables (#511) - -### 0.11.0 -- Oct 25, 2018 -- Full support of dependencies with renamed attributes using projection syntax (#300, #345, #436, #506, #507) -- Rename internal class and module names to comply with terminology in documentation (#494, #500) -- Full support of secondary indexes (#498, 500) -- ERD no longer shows numbers in nodes corresponding to derived dependencies (#478, #500) -- Full support of unique and nullable dependencies (#254, #301, #493, #495, #500) -- Improve memory management in `populate` (#461, #486) -- Fix query errors and redundancies (#456, #463, #482) - -### 0.10.1 -- Aug 28, 2018 -- Fix ERD Tooltip message (#431) -- Networkx 2.0 support (#443) -- Fix insert from query with skip_duplicates=True (#451) -- Sped up queries (#458) -- Bugfix in restriction of the form (A & B) \* B (#463) -- Improved error messages (#466) - -### 0.10.0 -- Jan 10, 2018 -- Deletes are more efficient (#424) -- ERD shows table definition on tooltip hover in Jupyter (#422) -- S3 external storage -- Garbage collection for external sorage -- Most operators and methods of tables can be invoked as class methods rather than instance methods (#407) -- The schema decorator object no longer requires locals() to specify the context -- Compatibility with pymysql 0.8.0+ -- More efficient loading of dependencies (#403) - -### 0.9.0 -- Nov 17, 2017 -- Made graphviz installation optional -- Implement file-based external storage -- Implement union operator + -- Implement file-based external storage - -### 0.8.0 -- Jul 26, 2017 -Documentation and tutorials available at https://docs.datajoint.io and https://tutorials.datajoint.io - -- improved the ERD graphics and features using the graphviz libraries (#207, #333) -- improved password handling logic (#322, #321) -- the use of the `contents` property to populate tables now only works in `dj.Lookup` classes (#310). -- allow suppressing the display of size of query results through the `show_tuple_count` configuration option (#309) -- implemented renamed foreign keys to spec (#333) -- added the `limit` keyword argument to populate (#329) -- reduced the number of displayed messages (#308) -- added `size_on_disk` property for dj.Schema() objects (#323) -- job keys are entered in the jobs table (#316, #243) -- simplified the `fetch` and `fetch1` syntax, deprecating the `fetch[...]` syntax (#319) -- the jobs tables now store the connection ids to allow identifying abandoned jobs (#288, #317) - -### 0.5.0 (#298) -- Mar 8, 2017 -- All fetched integers are now 64-bit long and all fetched floats are double precision. -- Added `dj.create_virtual_module` - -### 0.4.10 (#286) -- Feb 6, 2017 -- Removed Vagrant and Readthedocs support -- Explicit saving of configuration (issue #284) - -### 0.4.9 (#285) -- Feb 2, 2017 -- Fixed setup.py for pip install - -### 0.4.7 (#281) -- Jan 24, 2017 -- Fixed issues related to order of attributes in projection. - -### 0.4.6 (#277) -- Dec 22, 2016 -- Proper handling of interruptions during populate - -### 0.4.5 (#274) -- Dec 20, 2016 -- Populate reports how many keys remain to be populated at the start. - -### 0.4.3 (#271) -- Dec 6, 2016 -- Fixed aggregation issues (#270) -- datajoint no longer attempts to connect to server at import time -- dropped support of view (reversed #257) -- more elegant handling of insufficient privileges (#268) - -### 0.4.2 (#267) -- Dec 6, 2016 -- improved table appearance in Jupyter - -### 0.4.1 (#266) -- Oct 28, 2016 -- bugfix for very long error messages - -### 0.3.9 -- Sep 27, 2016 -- Added support for datatype `YEAR` -- Fixed issues with `dj.U` and the `aggr` operator (#246, #247) - -### 0.3.8 -- Aug 2, 2016 -- added the `_update` method in `base_relation`. It allows updating values in existing tuples. -- bugfix in reading values of type double. Previously it was cast as float32. - -### 0.3.7 -- Jul 31, 2016 -- added parameter `ignore_extra_fields` in `insert` -- `insert(..., skip_duplicates=True)` now relies on `SELECT IGNORE`. Previously it explicitly checked if tuple already exists. -- table previews now include blob attributes displaying the string - -### 0.3.6 -- Jul 30, 2016 -- bugfix in `schema.spawn_missing_classes`. Previously, spawned part classes would not show in ERDs. -- dj.key now causes fetch to return as a list of dicts. Previously it was a recarray. - -### 0.3.5 -- `dj.set_password()` now asks for user confirmation before changing the password. -- fixed issue #228 - -### 0.3.4 -- Added method the `ERD.add_parts` method, which adds the part tables of all tables currently in the ERD. -- `ERD() + arg` and `ERD() - arg` can now accept table classes as arg. - -### 0.3.3 -- Suppressed warnings (redirected them to logging). Previoiusly, scipy would throw warnings in ERD, for example. -- Added ERD.from_sequence as a shortcut to combining the ERDs of multiple sources -- ERD() no longer text the context argument. -- ERD.draw() now takes an optional context argument. By default uses the caller's locals. - -### 0.3.2 -- Fixed issue #223: `insert` can insert relations without fetching. -- ERD() now takes the `context` argument, which specifies in which context to look for classes. The default is taken from the argument (schema or table). -- ERD.draw() no longer has the `prefix` argument: class names are shown as found in the context. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 64ad3f6ab..6499889d2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,3 +186,10 @@ def insert(self, rows, *, replace=False): - **Don't:** Repeat function signature in description See [NumPy Docstring Guide](https://numpydoc.readthedocs.io/en/latest/format.html) for full reference. + +## Releases + +Releases are annotated tags whose messages are the release notes; there is no changelog file to +update in a pull request. A pull request's title states its user-visible change, since the +notes are written from the titles. See [RELEASE_MEMO.md](RELEASE_MEMO.md) for how a release is +cut. diff --git a/RELEASE_MEMO.md b/RELEASE_MEMO.md index 1a3623330..84880adde 100644 --- a/RELEASE_MEMO.md +++ b/RELEASE_MEMO.md @@ -8,98 +8,79 @@ --- -## Writing Release Notes +## Releases Are Annotated Tags -Good release notes help users understand what changed and whether they need to take action. +A release is an annotated tag `vX.Y.Z` whose message is the release notes. The tag is the +record of the release: `git show vX.Y.Z` or `git tag -n99 vX.Y.Z` prints its notes, and the +GitHub release mirrors them. There is no changelog file. -### Categories +### Cutting a Release -Organize changes into these categories (in order): +1. **Collect the changes** merged since the last release: -| Category | When to Use | Example | -|----------|-------------|---------| -| **BREAKING** | Changes that require user action | API changes, removed features | -| **Added** | New features | New methods, new options | -| **Changed** | Behavior changes (non-breaking) | Performance improvements, defaults | -| **Deprecated** | Features marked for removal | Old syntax warnings | -| **Fixed** | Bug fixes | Error corrections | -| **Security** | Security patches | Vulnerability fixes | - -### Format - -```markdown -## What's Changed - -### BREAKING CHANGES -- **`fetch()` removed** — Use `to_dicts()`, `to_pandas()`, or `to_arrays()` instead (#123) - -### Added -- New `to_polars()` method for Polars DataFrame output (#456) -- Support for custom codecs via `@codec` decorator (#789) - -### Changed -- Improved query performance for complex joins (2-3x faster) -- Default connection timeout increased to 30s - -### Fixed -- Fixed incorrect NULL handling in aggregations (#234) - -### Full Changelog -https://github.com/datajoint/datajoint-python/compare/v2.0.0...v2.1.0 -``` - -### Guidelines + ```bash + gh pr list --state merged --base master --search "merged:>YYYY-MM-DD" --json number,title + ``` -1. **Lead with breaking changes** — Users need to see these first -2. **Explain the "why"** — Not just what changed, but why it matters -3. **Link to PRs/issues** — For users who want details -4. **Use imperative mood** — "Add feature" not "Added feature" -5. **Be concise** — One line per change, details in PR +2. **Write `notes.md`.** The first line is one sentence that says what the release is about. + Then, after a blank line, the changes grouped by area (queries, diagrams, jobs, storage, the + backends, packaging, ...), each naming its pull request. Put anything users must act on + first. -### PR Labels + ```markdown + Deletes remove only the rows they reach, and jobs record when they finished. -The release drafter uses PR labels to categorize changes: + ## Deletes + - `(T & dj.Top(n)).delete()` deletes the rows the `Top` selects, not every matching row (#1234) -| Label | Category | -|-------|----------| -| `breaking` | 💥 Breaking Changes | -| `feature` | 🚀 Features | -| `enhancement` | ⚡️ Enhancements | -| `bug` | 🐛 Bug Fixes | -| `documentation` | 📝 Documentation | + ## Jobs + - `completed_time` is recorded to the millisecond (#1235) + ``` -Ensure PRs have appropriate labels before merging. +3. **Tag the release commit on `master` and push the tag:** ---- + ```bash + git tag -a vX.Y.Z --cleanup=verbatim -F notes.md && git push origin vX.Y.Z + ``` -## PyPI Release Process + `--cleanup=verbatim` keeps lines that start with `#`, such as Markdown headings, which git + would otherwise drop as comments. -### Steps +4. **The [Release workflow](.github/workflows/release.yaml) does the rest:** + - checks that the tag is annotated and on `master`; + - runs the lint and test workflows; + - builds the package at the tag; + - publishes it to PyPI; + - creates the GitHub release, with the tag's message as its notes and the wheel and sdist + attached; + - posts to Slack. -1. **Add labels to merged PRs** for release-drafter categorization -2. **Run "Manual Draft Release" workflow** on GitHub Actions -3. **Edit the draft release**: - - Set release name to `Release X.Y.Z` - - Set tag to `vX.Y.Z` - - Review and edit release notes -4. **Publish the release** — this creates the `vX.Y.Z` tag. -5. Automation will: - - Build from the tag and publish to PyPI - - Attach the wheel/sdist as release assets +If a step fails after the tag is pushed, fix the cause and run the workflow again from the tag: +Actions -> Release -> Run workflow -> "Use workflow from" the tag, with **testpypi** unchecked. +To publish a tag to TestPyPI only, run it the same way with **testpypi** checked. Never move or +delete a pushed tag: the version on PyPI and the GitHub release are tied to it. -### Version Note +Pull request titles still state the user-visible change, since they are what the notes are +written from. Labels are not used for release notes. -The release drafter computes the version from the previous tag. You may need to -**manually edit** the release name/tag for major version changes. +### Versions The package version is **derived from the git tag** at build time by [hatch-vcs](https://github.com/ofek/hatch-vcs) (see `[tool.hatch.version]` in `pyproject.toml`) — the tag is the single source of truth. There is **no -`version.py` value to bump** and **no follow-up version PR**: publishing the +`version.py` value to bump** and **no follow-up version PR**: pushing the `vX.Y.Z` tag is what sets the version everywhere (`pip install`, `pip show`, and `datajoint.__version__`). Do not hand-edit `src/datajoint/version.py`; it only re-exports the build-generated `_version.py`. +### Releases Before This Process + +Every earlier release that has notes was re-tagged as an annotated tag on the same commit, +with those notes as its message: from the former changelog for 0.3.2 through 0.14.3, and from +the GitHub release for later versions. Such a message ends with a `Retroactive:` line, and the +Release workflow never publishes a tag that has one. A few early versions had no notes +anywhere; their tags stay as they were. + --- ## Conda-Forge Release Process @@ -110,7 +91,7 @@ DataJoint has a [conda-forge feedstock](https://github.com/conda-forge/datajoint Conda-forge has **automated bots** that detect new PyPI releases and create PRs automatically: -1. **You publish to PyPI** (via the GitHub release workflow) +1. **You publish to PyPI** (by pushing the release tag) 2. **regro-cf-autotick-bot** detects the new version within ~24 hours 3. **Bot creates a PR** to the feedstock with updated version and hash 4. **Maintainers review and merge**