From ab526cbe32d1dc1d7d13958de15aa9808ee6ac5a Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 20 Oct 2026 17:55:04 -0500 Subject: [PATCH 1/5] ci(release): publish a release from its annotated tag A release is now an annotated tag vX.Y.Z on main whose message is the release notes. Pushing the tag runs the CD workflow, which checks that the tag is annotated and on main, runs the CI workflow as the test gate, checks that the built files carry the tag's version, publishes them, and creates the GitHub release with the tag's message as its notes. The tag push replaces the "release published" trigger. The workflow file keeps its name, cd.yml, and the publish job keeps the pypi environment, because PyPI trusted publishing is bound to both. Pull requests and pushes to main still build the package, and a release run is never cancelled by a newer push. CI can now be called from CD. Its concurrency group is named "CI" explicitly: in a called run, github.workflow is the caller's name, and the shared group would make GitHub cancel the run. .github/scripts/check_release_tag.sh is the same script datajoint-python uses. CONTRIBUTING describes how to cut a release. --- .github/CONTRIBUTING.md | 47 +++++++++++++++ .github/scripts/check_release_tag.sh | 49 ++++++++++++++++ .github/workflows/cd.yml | 86 ++++++++++++++++++++++++++-- .github/workflows/ci.yml | 8 ++- 4 files changed, 182 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/check_release_tag.sh diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b9c4f0a..e71f7c7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -75,3 +75,50 @@ You can build the docs only with: ```bash nox -s docs --non-interactive ``` + +# Releasing + +A release is an annotated tag `vX.Y.Z` on `main` whose message is the release +notes. There is no changelog file: `git show vX.Y.Z` prints a release's notes, +and its GitHub release shows the same text. + +1. Collect the pull requests merged since the last release (for the first + release, leave out `--search`): + + ```bash + gh pr list --state merged --base main --search "merged:>YYYY-MM-DD" --json number,title + ``` + +2. Write `notes.md`. The first line is one sentence that says what the release + is about. Then, after a blank line, the changes grouped by area, each naming + its pull request. Put anything users must act on first. + + ```markdown + Fetched figures report their title without downloading the file. + + ## Fetch + + - `PlotlyRef.title` reads the title from the stored metadata (#123) + ``` + +3. Tag the release commit on `main` and push the tag: + + ```bash + git tag -a vX.Y.Z --cleanup=verbatim -F notes.md && git push origin vX.Y.Z + ``` + + `--cleanup=verbatim` keeps lines that start with `#`, such as Markdown + headings, which git would otherwise drop as comments. + +Pushing the tag runs the [CD workflow](workflows/cd.yml). It checks that the tag +is annotated and on `main`, and refuses a lightweight tag (one made without +`-a`); runs the CI workflow; publishes the package built at the tag through PyPI +trusted publishing; and creates the GitHub release, with the tag's message as +its notes and the wheel and sdist attached. + +The version comes from the tag: hatch-vcs derives it at build time, so there is +no version number to edit, and the workflow checks that the built files carry +the tag's version. If a job fails after the tag is pushed, fix the cause and +re-run the failed jobs from the workflow run; a fix that needs a new commit is +released under the next version. Never move or delete a pushed tag: the version +on PyPI and the GitHub release are tied to it. diff --git a/.github/scripts/check_release_tag.sh b/.github/scripts/check_release_tag.sh new file mode 100755 index 0000000..9b1b440 --- /dev/null +++ b/.github/scripts/check_release_tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Decide whether a release tag can be published. +# +# check_release_tag.sh [] +# +# Fails on a tag that is not vX.Y.Z, on a lightweight tag, and on a tag whose +# commit is not on (default: master). Succeeds without publishing on a +# tag whose message has a "Retroactive:" line: such tags record past releases. +# Writes publish=true or publish=false to $GITHUB_OUTPUT when it is set. +set -euo pipefail + +tag="${1:?usage: check_release_tag.sh []}" +branch="${2:-master}" +output="${GITHUB_OUTPUT:-/dev/null}" + +fail() { + echo "::error::$1" >&2 + shift + for line in "$@"; do echo "$line" >&2; done + exit 1 +} + +[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "$tag is not a release tag (vX.Y.Z)." + +# actions/checkout can store an annotated tag as a lightweight ref, so fetch the +# tag object itself, and the branch the tag must be on. +git fetch --quiet --force origin "refs/tags/$tag:refs/tags/$tag" +git fetch --quiet --no-tags origin "+refs/heads/$branch:refs/remotes/origin/$branch" + +if [[ "$(git cat-file -t "refs/tags/$tag")" != "tag" ]]; then + fail "$tag is a lightweight tag. A release tag is annotated, with the release notes as its message:" \ + " git tag -d $tag" \ + " git tag -a $tag --cleanup=verbatim -F notes.md" \ + " git push --force origin refs/tags/$tag" +fi + +commit="$(git rev-parse "refs/tags/$tag^{commit}")" +if ! git merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then + fail "$tag points to $commit, which is not on $branch." +fi + +if git for-each-ref "refs/tags/$tag" --format='%(contents)' | grep -q '^Retroactive:'; then + echo "::notice::$tag records a past release (its message has a Retroactive: line); nothing to publish." + echo "publish=false" >> "$output" + exit 0 +fi + +echo "$tag is an annotated tag on $branch; publishing." +echo "publish=true" >> "$output" diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1b327bb..98b034e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,18 +1,26 @@ name: CD +# Pull requests and pushes to main build the package. A release is an annotated +# tag vX.Y.Z on main whose message is the release notes: pushing it checks the +# tag, runs the CI workflow, publishes the package built at the tag (hatch-vcs +# takes the version from the tag), and creates the GitHub release with the +# tag's message as its notes. +# +# PyPI trusted publishing is bound to this file's name and to the pypi +# environment, so neither can be renamed without updating PyPI. on: workflow_dispatch: pull_request: push: branches: - main - release: - types: - - published + tags: + - "v[0-9]+.[0-9]+.[0-9]+" concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # A newer push replaces a build in progress; a release is never cancelled. + cancel-in-progress: ${{ github.ref_type != 'tag' }} env: # Many color libraries just need this to be set to any value, but at least @@ -35,8 +43,31 @@ jobs: # pins the full version deliberately. - uses: hynek/build-and-inspect-python-package@v3.0.1 + check: + name: Check the release tag + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + outputs: + publish: ${{ steps.check.outputs.publish }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Check the tag + id: check + run: .github/scripts/check_release_tag.sh "$GITHUB_REF_NAME" main + + ci: + name: CI + needs: [check] + if: needs.check.outputs.publish == 'true' + uses: ./.github/workflows/ci.yml + secrets: inherit + publish: - needs: [dist] + needs: [dist, check, ci] name: Publish to PyPI environment: pypi permissions: @@ -44,7 +75,9 @@ jobs: attestations: write contents: read runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'published' + if: github.ref_type == 'tag' && needs.check.outputs.publish == 'true' + env: + TAG: ${{ github.ref_name }} steps: - uses: actions/download-artifact@v8 @@ -52,6 +85,16 @@ jobs: name: Packages path: dist + - name: Check the built version + run: | + version="${TAG#v}" + for file in dist/*; do + case "$(basename "$file")" in + "dj_plotly_codecs-$version-"*.whl | "dj_plotly_codecs-$version.tar.gz") ;; + *) echo "::error::$file does not carry version $version"; exit 1 ;; + esac + done + - name: Generate artifact attestation for sdist and wheel uses: actions/attest-build-provenance@v4 with: @@ -62,3 +105,34 @@ jobs: # Remember to tell (test-)pypi about this repo before publishing # Remove this line to publish to PyPI repository-url: https://test.pypi.org/legacy/ + + release: + needs: [publish] + name: Create the GitHub release + runs-on: ubuntu-latest + permissions: + contents: write + env: + TAG: ${{ github.ref_name }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Fetch the tag object + # actions/checkout can store an annotated tag as a lightweight ref. + run: git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG" + + - uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + git for-each-ref "refs/tags/$TAG" --format='%(contents)' > "$RUNNER_TEMP/notes.md" + gh release create "$TAG" --verify-tag --title "dj-plotly-codecs ${TAG#v}" \ + --notes-file "$RUNNER_TEMP/notes.md" dist/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99c293d..04de274 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,14 @@ on: push: branches: - main + # The CD workflow runs this as the test gate of a release. + workflow_call: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # "CI" rather than github.workflow, which in a run called from CD is "CD": + # sharing the caller's group is a deadlock, and GitHub cancels the run. + group: CI-${{ github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} env: # Many color libraries just need this variable to be set to any value. From 8958050e823535d619cbec59e322dff4daff9823 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 20 Oct 2026 17:55:25 -0500 Subject: [PATCH 2/5] ci(cd): publish to PyPI, not TestPyPI The publish step no longer sets repository-url, so a release tag publishes to pypi.org. Trusted publishing is unchanged: the same workflow file and the same pypi environment. --- .github/workflows/cd.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 98b034e..59396f3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -101,10 +101,6 @@ jobs: subject-path: "dist/*" - uses: pypa/gh-action-pypi-publish@release/v1 - with: - # Remember to tell (test-)pypi about this repo before publishing - # Remove this line to publish to PyPI - repository-url: https://test.pypi.org/legacy/ release: needs: [publish] From a1dd3bf66a5de924f19a1e3787dfd6d7d81859c3 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 20 Oct 2026 18:00:33 -0500 Subject: [PATCH 3/5] build: require datajoint 2.3.4 The codec calls self._extract_context(key, context) and self._codec_config(key, context), which datajoint adds in 2.3.4. Under 2.3.0 to 2.3.3, which datajoint>=2.3 allowed, _extract_context takes only the key and _codec_config does not exist, so every encode and decode would fail. The README and the docs index state the requirement. uv.lock already locked datajoint 2.3.4; only its recorded specifier changes. --- README.md | 2 ++ docs/index.md | 2 ++ pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5398b5..19e94d8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON. pip install "dj-plotly-codecs[plotly]" ``` +Requires Python 3.10 or later and `datajoint>=2.3.4`. + Without the `plotly` extra, the codec stores and loads dict figures, without validation and without `Figure` objects. diff --git a/docs/index.md b/docs/index.md index e5398b5..19e94d8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON. pip install "dj-plotly-codecs[plotly]" ``` +Requires Python 3.10 or later and `datajoint>=2.3.4`. + Without the `plotly` extra, the codec stores and loads dict figures, without validation and without `Figure` objects. diff --git a/pyproject.toml b/pyproject.toml index fe17064..d80cdb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "datajoint>=2.3", + "datajoint>=2.3.4", "numpy>=2.0", ] diff --git a/uv.lock b/uv.lock index aa48027..af06af4 100644 --- a/uv.lock +++ b/uv.lock @@ -579,7 +579,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "datajoint", specifier = ">=2.3" }, + { name = "datajoint", specifier = ">=2.3.4" }, { name = "numpy", specifier = ">=2.0" }, { name = "plotly", marker = "extra == 'plotly'", specifier = ">=5" }, ] From 845b8986cf74f955b3594cd643cd168036bd1cdb Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 20 Oct 2026 18:04:49 -0500 Subject: [PATCH 4/5] fix(codec): read schema, table, field and config from the context only DataJoint 2.3.4 and later pass context to every codec that declares it: schema, table, field and config on insert, config on fetch. The codec now reads them there and nowhere else. It no longer calls _extract_context and _codec_config, which fall back to the underscore keys in key (_schema, _table, _field, _config) when context lacks a value; DataJoint 2.4 removes those keys. The primary key still comes from key, without its underscore entries. Called without a context, or with one that lacks a value it needs, the codec raises a DataJointError that names the missing values and says that DataJoint 2.3.4 and later pass them, before it reads or writes any file. New tests call encode and decode without a context, with key filled the way DataJoint 2.3 fills it, and expect that error. The round trips through a real table check that DataJoint 2.3.4 passes the context on insert and on fetch. --- src/dj_plotly_codecs/codecs.py | 53 +++++++++++++++++++++++++++++----- tests/test_codec.py | 30 +++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/dj_plotly_codecs/codecs.py b/src/dj_plotly_codecs/codecs.py index e441fdb..b14a819 100644 --- a/src/dj_plotly_codecs/codecs.py +++ b/src/dj_plotly_codecs/codecs.py @@ -293,6 +293,29 @@ def __repr__(self) -> str: ) +def _require_context(context: dict[str, Any] | None, *names: str) -> dict[str, Any]: + """ + Return ``context``, checked to carry ``names``. + + DataJoint 2.3.4 and later pass ``context`` to every codec that declares it: + ``schema``, ``table``, ``field``, and ``config`` on insert, ``config`` on + fetch. The codec reads them there only, never from underscore keys in ``key``. + """ + missing = [n for n in names if context is None or context.get(n) is None] + if context is None or missing: + msg = ( + " needs the context that DataJoint 2.3.4 and later pass to codecs " + f"(missing: {', '.join(missing)}); install datajoint>=2.3.4" + ) + raise DataJointError(msg) + return context + + +def _primary_key(key: dict[str, Any] | None) -> dict[str, Any]: + """The primary key values in ``key``, without the underscore keys DataJoint 2.3 adds.""" + return {k: v for k, v in (key or {}).items() if not k.startswith("_")} + + class PlotlyCodec(SchemaCodec): # type: ignore[misc] """ Schema-addressed storage for Plotly figures as ``.plotly.json`` files. @@ -373,15 +396,21 @@ def encode( ------- dict JSON metadata: ``{path, store, size, traces, trace_types, title}``. + + Raises + ------ + DataJointError + If ``context`` lacks the ``schema``, ``table``, ``field``, or + ``config`` that DataJoint 2.3.4 and later pass. """ + context = _require_context(context, "schema", "table", "field", "config") + config = context["config"] data = encode_plotly(value) - schema, table, field, primary_key = self._extract_context(key, context) - config = self._codec_config(key, context) path, _ = self._build_path( - schema, - table, - field, - primary_key, + context["schema"], + context["table"], + context["field"], + _primary_key(key), ext=EXTENSION, store_name=store_name, config=config, @@ -404,7 +433,15 @@ def decode( key: dict[str, Any] | None = None, context: dict[str, Any] | None = None, ) -> PlotlyRef: - """Create a lazy PlotlyRef from stored metadata.""" - config = self._codec_config(key, context) + """ + Create a lazy PlotlyRef from stored metadata. + + Raises + ------ + DataJointError + If ``context`` lacks the ``config`` that DataJoint 2.3.4 and later pass. + """ + del key # the stored metadata locates the file + config = _require_context(context, "config")["config"] backend = self._get_backend(stored.get("store"), config=config) return PlotlyRef(stored, backend) diff --git a/tests/test_codec.py b/tests/test_codec.py index cb795b8..6592c1b 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -218,6 +218,36 @@ def no_plotly(name: str, *args: Any, **kwargs: Any) -> Any: ref.to_figure() +# The codec context, which DataJoint 2.3.4 and later pass + +# How DataJoint 2.3 still fills `key`; the codec does not read the underscore keys. +LEGACY_KEY = { + "id": 1, + "_schema": "s", + "_table": "t", + "_field": "f", + "_config": dj.config, +} + + +@pytest.mark.parametrize( + "context", + [None, {"schema": "s", "table": "t", "field": "f"}], + ids=["no context", "no config"], +) +def test_encode_needs_context(context: dict[str, Any] | None) -> None: + with pytest.raises(dj.DataJointError, match=r"DataJoint 2\.3\.4"): + PlotlyCodec().encode( + _figure(), key=LEGACY_KEY, context=context, store_name="store" + ) + + +def test_decode_needs_context() -> None: + stored = {"path": "f.plotly.json", "store": None, "size": 1, "traces": 0} + with pytest.raises(dj.DataJointError, match=r"missing: config"): + PlotlyCodec().decode(stored, key={"_config": dj.config}, context=None) + + # Through DataJoint, with a database and a file store From 7434199b1bcb8f447fbd947b300828ab851ff22c Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 20 Oct 2026 18:05:44 -0500 Subject: [PATCH 5/5] ci: run the Format job on Python 3.14 The job set up "3.x", which now resolves to Python 3.15. Its pylint session installs the package and so datajoint, whose dependency deepdiff resolves to 9.1, which needs cachebox 5. cachebox 5 has no Python 3.15 build and fails to compile, so the job fails. It runs on 3.14 until datajoint 2.3.5, which supports Python 3.15 and keeps deepdiff below 9 there, is on PyPI. No other job resolves to 3.15: the test matrix names its versions, and the build action in CD defaults to 3.14. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 04de274..5120848 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,11 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v7 with: - python-version: "3.x" + # Not "3.x", which now resolves to 3.15: the pylint session installs + # datajoint, whose dependency deepdiff 9.1 needs cachebox 5, which has + # no Python 3.15 build. Return to "3.x" once datajoint 2.3.5 is on PyPI: + # it supports Python 3.15 and keeps deepdiff below 9 there. + python-version: "3.14" - uses: astral-sh/setup-uv@v7