ci(release): publish to PyPI from an annotated tag

.github/CONTRIBUTING.md

@@ -75,3 +75,50 @@ You can build the docs only with:
 ```bash
 nox -s docs --non-interactive
 ```
+
+# Releasing
+
+A release is an annotated tag `vX.Y.Z` on `main` whose message is the release
+notes. There is no changelog file: `git show vX.Y.Z` prints a release's notes,
+and its GitHub release shows the same text.
+
+1. Collect the pull requests merged since the last release (for the first
+   release, leave out `--search`):
+
+   ```bash
+   gh pr list --state merged --base main --search "merged:>YYYY-MM-DD" --json number,title
+   ```
+
+2. Write `notes.md`. The first line is one sentence that says what the release
+   is about. Then, after a blank line, the changes grouped by area, each naming
+   its pull request. Put anything users must act on first.
+
+   ```markdown
+   Fetched figures report their title without downloading the file.
+
+   ## Fetch
+
+   - `PlotlyRef.title` reads the title from the stored metadata (#123)
+   ```
+
+3. Tag the release commit on `main` and push the tag:
+
+   ```bash
+   git tag -a vX.Y.Z --cleanup=verbatim -F notes.md && git push origin vX.Y.Z
+   ```
+
+   `--cleanup=verbatim` keeps lines that start with `#`, such as Markdown
+   headings, which git would otherwise drop as comments.
+
+Pushing the tag runs the [CD workflow](workflows/cd.yml). It checks that the tag
+is annotated and on `main`, and refuses a lightweight tag (one made without
+`-a`); runs the CI workflow; publishes the package built at the tag through PyPI
+trusted publishing; and creates the GitHub release, with the tag's message as
+its notes and the wheel and sdist attached.
+
+The version comes from the tag: hatch-vcs derives it at build time, so there is
+no version number to edit, and the workflow checks that the built files carry
+the tag's version. If a job fails after the tag is pushed, fix the cause and
+re-run the failed jobs from the workflow run; a fix that needs a new commit is
+released under the next version. Never move or delete a pushed tag: the version
+on PyPI and the GitHub release are tied to it.

.github/scripts/check_release_tag.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Decide whether a release tag can be published.
+#
+#   check_release_tag.sh <tag> [<branch>]
+#
+# Fails on a tag that is not vX.Y.Z, on a lightweight tag, and on a tag whose
+# commit is not on <branch> (default: master). Succeeds without publishing on a
+# tag whose message has a "Retroactive:" line: such tags record past releases.
+# Writes publish=true or publish=false to $GITHUB_OUTPUT when it is set.
+set -euo pipefail
+
+tag="${1:?usage: check_release_tag.sh <tag> [<branch>]}"
+branch="${2:-master}"
+output="${GITHUB_OUTPUT:-/dev/null}"
+
+fail() {
+  echo "::error::$1" >&2
+  shift
+  for line in "$@"; do echo "$line" >&2; done
+  exit 1
+}
+
+[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "$tag is not a release tag (vX.Y.Z)."
+
+# actions/checkout can store an annotated tag as a lightweight ref, so fetch the
+# tag object itself, and the branch the tag must be on.
+git fetch --quiet --force origin "refs/tags/$tag:refs/tags/$tag"
+git fetch --quiet --no-tags origin "+refs/heads/$branch:refs/remotes/origin/$branch"
+
+if [[ "$(git cat-file -t "refs/tags/$tag")" != "tag" ]]; then
+  fail "$tag is a lightweight tag. A release tag is annotated, with the release notes as its message:" \
+    "  git tag -d $tag" \
+    "  git tag -a $tag <commit> --cleanup=verbatim -F notes.md" \
+    "  git push --force origin refs/tags/$tag"
+fi
+
+commit="$(git rev-parse "refs/tags/$tag^{commit}")"
+if ! git merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then
+  fail "$tag points to $commit, which is not on $branch."
+fi
+
+if git for-each-ref "refs/tags/$tag" --format='%(contents)' | grep -q '^Retroactive:'; then
+  echo "::notice::$tag records a past release (its message has a Retroactive: line); nothing to publish."
+  echo "publish=false" >> "$output"
+  exit 0
+fi
+
+echo "$tag is an annotated tag on $branch; publishing."
+echo "publish=true" >> "$output"

.github/workflows/cd.yml

@@ -1,18 +1,26 @@
 name: CD
 
+# Pull requests and pushes to main build the package. A release is an annotated
+# tag vX.Y.Z on main whose message is the release notes: pushing it checks the
+# tag, runs the CI workflow, publishes the package built at the tag (hatch-vcs
+# takes the version from the tag), and creates the GitHub release with the
+# tag's message as its notes.
+#
+# PyPI trusted publishing is bound to this file's name and to the pypi
+# environment, so neither can be renamed without updating PyPI.
 on:
   workflow_dispatch:
   pull_request:
   push:
     branches:
       - main
-  release:
-    types:
-      - published
+    tags:
+      - "v[0-9]+.[0-9]+.[0-9]+"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  # A newer push replaces a build in progress; a release is never cancelled.
+  cancel-in-progress: ${{ github.ref_type != 'tag' }}
 
 env:
   # Many color libraries just need this to be set to any value, but at least
@@ -35,30 +43,92 @@ jobs:
       # pins the full version deliberately.
       - uses: hynek/build-and-inspect-python-package@v3.0.1
 
+  check:
+    name: Check the release tag
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    outputs:
+      publish: ${{ steps.check.outputs.publish }}
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Check the tag
+        id: check
+        run: .github/scripts/check_release_tag.sh "$GITHUB_REF_NAME" main
+
+  ci:
+    name: CI
+    needs: [check]
+    if: needs.check.outputs.publish == 'true'
+    uses: ./.github/workflows/ci.yml
+    secrets: inherit
+
   publish:
-    needs: [dist]
+    needs: [dist, check, ci]
     name: Publish to PyPI
     environment: pypi
     permissions:
       id-token: write
       attestations: write
       contents: read
     runs-on: ubuntu-latest
-    if: github.event_name == 'release' && github.event.action == 'published'
+    if: github.ref_type == 'tag' && needs.check.outputs.publish == 'true'
+    env:
+      TAG: ${{ github.ref_name }}
 
     steps:
       - uses: actions/download-artifact@v8
         with:
           name: Packages
           path: dist
 
+      - name: Check the built version
+        run: |
+          version="${TAG#v}"
+          for file in dist/*; do
+            case "$(basename "$file")" in
+              "dj_plotly_codecs-$version-"*.whl | "dj_plotly_codecs-$version.tar.gz") ;;
+              *) echo "::error::$file does not carry version $version"; exit 1 ;;
+            esac
+          done
+
       - name: Generate artifact attestation for sdist and wheel
         uses: actions/attest-build-provenance@v4
         with:
           subject-path: "dist/*"
 
       - uses: pypa/gh-action-pypi-publish@release/v1
+
+  release:
+    needs: [publish]
+    name: Create the GitHub release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      TAG: ${{ github.ref_name }}
+
+    steps:
+      - uses: actions/checkout@v7
         with:
-          # Remember to tell (test-)pypi about this repo before publishing
-          # Remove this line to publish to PyPI
-          repository-url: https://test.pypi.org/legacy/
+          fetch-depth: 0
+
+      - name: Fetch the tag object
+        # actions/checkout can store an annotated tag as a lightweight ref.
+        run: git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG"
+
+      - uses: actions/download-artifact@v8
+        with:
+          name: Packages
+          path: dist
+
+      - name: Create the GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          git for-each-ref "refs/tags/$TAG" --format='%(contents)' > "$RUNNER_TEMP/notes.md"
+          gh release create "$TAG" --verify-tag --title "dj-plotly-codecs ${TAG#v}" \
+            --notes-file "$RUNNER_TEMP/notes.md" dist/*

.github/workflows/ci.yml

@@ -6,10 +6,14 @@ on:
   push:
     branches:
       - main
+  # The CD workflow runs this as the test gate of a release.
+  workflow_call:
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+  # "CI" rather than github.workflow, which in a run called from CD is "CD":
+  # sharing the caller's group is a deadlock, and GitHub cancels the run.
+  group: CI-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref_type != 'tag' }}
 
 env:
   # Many color libraries just need this variable to be set to any value.
@@ -27,7 +31,11 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v7
         with:
-          python-version: "3.x"
+          # Not "3.x", which now resolves to 3.15: the pylint session installs
+          # datajoint, whose dependency deepdiff 9.1 needs cachebox 5, which has
+          # no Python 3.15 build. Return to "3.x" once datajoint 2.3.5 is on PyPI:
+          # it supports Python 3.15 and keeps deepdiff below 9 there.
+          python-version: "3.14"
 
       - uses: astral-sh/setup-uv@v7
 

README.md

@@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON.
 pip install "dj-plotly-codecs[plotly]"
 ```
 
+Requires Python 3.10 or later and `datajoint>=2.3.4`.
+
 Without the `plotly` extra, the codec stores and loads dict figures, without
 validation and without `Figure` objects.
 

docs/index.md

@@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON.
 pip install "dj-plotly-codecs[plotly]"
 ```
 
+Requires Python 3.10 or later and `datajoint>=2.3.4`.
+
 Without the `plotly` extra, the codec stores and loads dict figures, without
 validation and without `Figure` objects.
 

pyproject.toml

@@ -31,7 +31,7 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = [
-    "datajoint>=2.3",
+    "datajoint>=2.3.4",
     "numpy>=2.0",
 ]
 

src/dj_plotly_codecs/codecs.py

@@ -293,6 +293,29 @@ def __repr__(self) -> str:
         )
 
 
+def _require_context(context: dict[str, Any] | None, *names: str) -> dict[str, Any]:
+    """
+    Return ``context``, checked to carry ``names``.
+
+    DataJoint 2.3.4 and later pass ``context`` to every codec that declares it:
+    ``schema``, ``table``, ``field``, and ``config`` on insert, ``config`` on
+    fetch. The codec reads them there only, never from underscore keys in ``key``.
+    """
+    missing = [n for n in names if context is None or context.get(n) is None]
+    if context is None or missing:
+        msg = (
+            "<plotly> needs the context that DataJoint 2.3.4 and later pass to codecs "
+            f"(missing: {', '.join(missing)}); install datajoint>=2.3.4"
+        )
+        raise DataJointError(msg)
+    return context
+
+
+def _primary_key(key: dict[str, Any] | None) -> dict[str, Any]:
+    """The primary key values in ``key``, without the underscore keys DataJoint 2.3 adds."""
+    return {k: v for k, v in (key or {}).items() if not k.startswith("_")}
+
+
 class PlotlyCodec(SchemaCodec):  # type: ignore[misc]
     """
     Schema-addressed storage for Plotly figures as ``.plotly.json`` files.
@@ -373,15 +396,21 @@ def encode(
         -------
         dict
             JSON metadata: ``{path, store, size, traces, trace_types, title}``.
+
+        Raises
+        ------
+        DataJointError
+            If ``context`` lacks the ``schema``, ``table``, ``field``, or
+            ``config`` that DataJoint 2.3.4 and later pass.
         """
+        context = _require_context(context, "schema", "table", "field", "config")
+        config = context["config"]
         data = encode_plotly(value)
-        schema, table, field, primary_key = self._extract_context(key, context)
-        config = self._codec_config(key, context)
         path, _ = self._build_path(
-            schema,
-            table,
-            field,
-            primary_key,
+            context["schema"],
+            context["table"],
+            context["field"],
+            _primary_key(key),
             ext=EXTENSION,
             store_name=store_name,
             config=config,
@@ -404,7 +433,15 @@ def decode(
         key: dict[str, Any] | None = None,
         context: dict[str, Any] | None = None,
     ) -> PlotlyRef:
-        """Create a lazy PlotlyRef from stored metadata."""
-        config = self._codec_config(key, context)
+        """
+        Create a lazy PlotlyRef from stored metadata.
+
+        Raises
+        ------
+        DataJointError
+            If ``context`` lacks the ``config`` that DataJoint 2.3.4 and later pass.
+        """
+        del key  # the stored metadata locates the file
+        config = _require_context(context, "config")["config"]
         backend = self._get_backend(stored.get("store"), config=config)
         return PlotlyRef(stored, backend)

tests/test_codec.py

@@ -218,6 +218,36 @@ def no_plotly(name: str, *args: Any, **kwargs: Any) -> Any:
         ref.to_figure()
 
 
+# The codec context, which DataJoint 2.3.4 and later pass
+
+# How DataJoint 2.3 still fills `key`; the codec does not read the underscore keys.
+LEGACY_KEY = {
+    "id": 1,
+    "_schema": "s",
+    "_table": "t",
+    "_field": "f",
+    "_config": dj.config,
+}
+
+
+@pytest.mark.parametrize(
+    "context",
+    [None, {"schema": "s", "table": "t", "field": "f"}],
+    ids=["no context", "no config"],
+)
+def test_encode_needs_context(context: dict[str, Any] | None) -> None:
+    with pytest.raises(dj.DataJointError, match=r"DataJoint 2\.3\.4"):
+        PlotlyCodec().encode(
+            _figure(), key=LEGACY_KEY, context=context, store_name="store"
+        )
+
+
+def test_decode_needs_context() -> None:
+    stored = {"path": "f.plotly.json", "store": None, "size": 1, "traces": 0}
+    with pytest.raises(dj.DataJointError, match=r"missing: config"):
+        PlotlyCodec().decode(stored, key={"_config": dj.config}, context=None)
+
+
 # Through DataJoint, with a database and a file store