diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index b9c4f0a..e71f7c7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -75,3 +75,50 @@ You can build the docs only with: ```bash nox -s docs --non-interactive ``` + +# Releasing + +A release is an annotated tag `vX.Y.Z` on `main` whose message is the release +notes. There is no changelog file: `git show vX.Y.Z` prints a release's notes, +and its GitHub release shows the same text. + +1. Collect the pull requests merged since the last release (for the first + release, leave out `--search`): + + ```bash + gh pr list --state merged --base main --search "merged:>YYYY-MM-DD" --json number,title + ``` + +2. Write `notes.md`. The first line is one sentence that says what the release + is about. Then, after a blank line, the changes grouped by area, each naming + its pull request. Put anything users must act on first. + + ```markdown + Fetched figures report their title without downloading the file. + + ## Fetch + + - `PlotlyRef.title` reads the title from the stored metadata (#123) + ``` + +3. Tag the release commit on `main` and push the tag: + + ```bash + git tag -a vX.Y.Z --cleanup=verbatim -F notes.md && git push origin vX.Y.Z + ``` + + `--cleanup=verbatim` keeps lines that start with `#`, such as Markdown + headings, which git would otherwise drop as comments. + +Pushing the tag runs the [CD workflow](workflows/cd.yml). It checks that the tag +is annotated and on `main`, and refuses a lightweight tag (one made without +`-a`); runs the CI workflow; publishes the package built at the tag through PyPI +trusted publishing; and creates the GitHub release, with the tag's message as +its notes and the wheel and sdist attached. + +The version comes from the tag: hatch-vcs derives it at build time, so there is +no version number to edit, and the workflow checks that the built files carry +the tag's version. If a job fails after the tag is pushed, fix the cause and +re-run the failed jobs from the workflow run; a fix that needs a new commit is +released under the next version. Never move or delete a pushed tag: the version +on PyPI and the GitHub release are tied to it. diff --git a/.github/scripts/check_release_tag.sh b/.github/scripts/check_release_tag.sh new file mode 100755 index 0000000..9b1b440 --- /dev/null +++ b/.github/scripts/check_release_tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Decide whether a release tag can be published. +# +# check_release_tag.sh [] +# +# Fails on a tag that is not vX.Y.Z, on a lightweight tag, and on a tag whose +# commit is not on (default: master). Succeeds without publishing on a +# tag whose message has a "Retroactive:" line: such tags record past releases. +# Writes publish=true or publish=false to $GITHUB_OUTPUT when it is set. +set -euo pipefail + +tag="${1:?usage: check_release_tag.sh []}" +branch="${2:-master}" +output="${GITHUB_OUTPUT:-/dev/null}" + +fail() { + echo "::error::$1" >&2 + shift + for line in "$@"; do echo "$line" >&2; done + exit 1 +} + +[[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "$tag is not a release tag (vX.Y.Z)." + +# actions/checkout can store an annotated tag as a lightweight ref, so fetch the +# tag object itself, and the branch the tag must be on. +git fetch --quiet --force origin "refs/tags/$tag:refs/tags/$tag" +git fetch --quiet --no-tags origin "+refs/heads/$branch:refs/remotes/origin/$branch" + +if [[ "$(git cat-file -t "refs/tags/$tag")" != "tag" ]]; then + fail "$tag is a lightweight tag. A release tag is annotated, with the release notes as its message:" \ + " git tag -d $tag" \ + " git tag -a $tag --cleanup=verbatim -F notes.md" \ + " git push --force origin refs/tags/$tag" +fi + +commit="$(git rev-parse "refs/tags/$tag^{commit}")" +if ! git merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then + fail "$tag points to $commit, which is not on $branch." +fi + +if git for-each-ref "refs/tags/$tag" --format='%(contents)' | grep -q '^Retroactive:'; then + echo "::notice::$tag records a past release (its message has a Retroactive: line); nothing to publish." + echo "publish=false" >> "$output" + exit 0 +fi + +echo "$tag is an annotated tag on $branch; publishing." +echo "publish=true" >> "$output" diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1b327bb..59396f3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,18 +1,26 @@ name: CD +# Pull requests and pushes to main build the package. A release is an annotated +# tag vX.Y.Z on main whose message is the release notes: pushing it checks the +# tag, runs the CI workflow, publishes the package built at the tag (hatch-vcs +# takes the version from the tag), and creates the GitHub release with the +# tag's message as its notes. +# +# PyPI trusted publishing is bound to this file's name and to the pypi +# environment, so neither can be renamed without updating PyPI. on: workflow_dispatch: pull_request: push: branches: - main - release: - types: - - published + tags: + - "v[0-9]+.[0-9]+.[0-9]+" concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # A newer push replaces a build in progress; a release is never cancelled. + cancel-in-progress: ${{ github.ref_type != 'tag' }} env: # Many color libraries just need this to be set to any value, but at least @@ -35,8 +43,31 @@ jobs: # pins the full version deliberately. - uses: hynek/build-and-inspect-python-package@v3.0.1 + check: + name: Check the release tag + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + outputs: + publish: ${{ steps.check.outputs.publish }} + + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Check the tag + id: check + run: .github/scripts/check_release_tag.sh "$GITHUB_REF_NAME" main + + ci: + name: CI + needs: [check] + if: needs.check.outputs.publish == 'true' + uses: ./.github/workflows/ci.yml + secrets: inherit + publish: - needs: [dist] + needs: [dist, check, ci] name: Publish to PyPI environment: pypi permissions: @@ -44,7 +75,9 @@ jobs: attestations: write contents: read runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'published' + if: github.ref_type == 'tag' && needs.check.outputs.publish == 'true' + env: + TAG: ${{ github.ref_name }} steps: - uses: actions/download-artifact@v8 @@ -52,13 +85,50 @@ jobs: name: Packages path: dist + - name: Check the built version + run: | + version="${TAG#v}" + for file in dist/*; do + case "$(basename "$file")" in + "dj_plotly_codecs-$version-"*.whl | "dj_plotly_codecs-$version.tar.gz") ;; + *) echo "::error::$file does not carry version $version"; exit 1 ;; + esac + done + - name: Generate artifact attestation for sdist and wheel uses: actions/attest-build-provenance@v4 with: subject-path: "dist/*" - uses: pypa/gh-action-pypi-publish@release/v1 + + release: + needs: [publish] + name: Create the GitHub release + runs-on: ubuntu-latest + permissions: + contents: write + env: + TAG: ${{ github.ref_name }} + + steps: + - uses: actions/checkout@v7 with: - # Remember to tell (test-)pypi about this repo before publishing - # Remove this line to publish to PyPI - repository-url: https://test.pypi.org/legacy/ + fetch-depth: 0 + + - name: Fetch the tag object + # actions/checkout can store an annotated tag as a lightweight ref. + run: git fetch --force origin "refs/tags/$TAG:refs/tags/$TAG" + + - uses: actions/download-artifact@v8 + with: + name: Packages + path: dist + + - name: Create the GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + git for-each-ref "refs/tags/$TAG" --format='%(contents)' > "$RUNNER_TEMP/notes.md" + gh release create "$TAG" --verify-tag --title "dj-plotly-codecs ${TAG#v}" \ + --notes-file "$RUNNER_TEMP/notes.md" dist/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99c293d..5120848 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,10 +6,14 @@ on: push: branches: - main + # The CD workflow runs this as the test gate of a release. + workflow_call: concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # "CI" rather than github.workflow, which in a run called from CD is "CD": + # sharing the caller's group is a deadlock, and GitHub cancels the run. + group: CI-${{ github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} env: # Many color libraries just need this variable to be set to any value. @@ -27,7 +31,11 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v7 with: - python-version: "3.x" + # Not "3.x", which now resolves to 3.15: the pylint session installs + # datajoint, whose dependency deepdiff 9.1 needs cachebox 5, which has + # no Python 3.15 build. Return to "3.x" once datajoint 2.3.5 is on PyPI: + # it supports Python 3.15 and keeps deepdiff below 9 there. + python-version: "3.14" - uses: astral-sh/setup-uv@v7 diff --git a/README.md b/README.md index e5398b5..19e94d8 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON. pip install "dj-plotly-codecs[plotly]" ``` +Requires Python 3.10 or later and `datajoint>=2.3.4`. + Without the `plotly` extra, the codec stores and loads dict figures, without validation and without `Figure` objects. diff --git a/docs/index.md b/docs/index.md index e5398b5..19e94d8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,6 +30,8 @@ type tell tools that a column holds figures, not arbitrary JSON. pip install "dj-plotly-codecs[plotly]" ``` +Requires Python 3.10 or later and `datajoint>=2.3.4`. + Without the `plotly` extra, the codec stores and loads dict figures, without validation and without `Figure` objects. diff --git a/pyproject.toml b/pyproject.toml index fe17064..d80cdb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "datajoint>=2.3", + "datajoint>=2.3.4", "numpy>=2.0", ] diff --git a/src/dj_plotly_codecs/codecs.py b/src/dj_plotly_codecs/codecs.py index e441fdb..b14a819 100644 --- a/src/dj_plotly_codecs/codecs.py +++ b/src/dj_plotly_codecs/codecs.py @@ -293,6 +293,29 @@ def __repr__(self) -> str: ) +def _require_context(context: dict[str, Any] | None, *names: str) -> dict[str, Any]: + """ + Return ``context``, checked to carry ``names``. + + DataJoint 2.3.4 and later pass ``context`` to every codec that declares it: + ``schema``, ``table``, ``field``, and ``config`` on insert, ``config`` on + fetch. The codec reads them there only, never from underscore keys in ``key``. + """ + missing = [n for n in names if context is None or context.get(n) is None] + if context is None or missing: + msg = ( + " needs the context that DataJoint 2.3.4 and later pass to codecs " + f"(missing: {', '.join(missing)}); install datajoint>=2.3.4" + ) + raise DataJointError(msg) + return context + + +def _primary_key(key: dict[str, Any] | None) -> dict[str, Any]: + """The primary key values in ``key``, without the underscore keys DataJoint 2.3 adds.""" + return {k: v for k, v in (key or {}).items() if not k.startswith("_")} + + class PlotlyCodec(SchemaCodec): # type: ignore[misc] """ Schema-addressed storage for Plotly figures as ``.plotly.json`` files. @@ -373,15 +396,21 @@ def encode( ------- dict JSON metadata: ``{path, store, size, traces, trace_types, title}``. + + Raises + ------ + DataJointError + If ``context`` lacks the ``schema``, ``table``, ``field``, or + ``config`` that DataJoint 2.3.4 and later pass. """ + context = _require_context(context, "schema", "table", "field", "config") + config = context["config"] data = encode_plotly(value) - schema, table, field, primary_key = self._extract_context(key, context) - config = self._codec_config(key, context) path, _ = self._build_path( - schema, - table, - field, - primary_key, + context["schema"], + context["table"], + context["field"], + _primary_key(key), ext=EXTENSION, store_name=store_name, config=config, @@ -404,7 +433,15 @@ def decode( key: dict[str, Any] | None = None, context: dict[str, Any] | None = None, ) -> PlotlyRef: - """Create a lazy PlotlyRef from stored metadata.""" - config = self._codec_config(key, context) + """ + Create a lazy PlotlyRef from stored metadata. + + Raises + ------ + DataJointError + If ``context`` lacks the ``config`` that DataJoint 2.3.4 and later pass. + """ + del key # the stored metadata locates the file + config = _require_context(context, "config")["config"] backend = self._get_backend(stored.get("store"), config=config) return PlotlyRef(stored, backend) diff --git a/tests/test_codec.py b/tests/test_codec.py index cb795b8..6592c1b 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -218,6 +218,36 @@ def no_plotly(name: str, *args: Any, **kwargs: Any) -> Any: ref.to_figure() +# The codec context, which DataJoint 2.3.4 and later pass + +# How DataJoint 2.3 still fills `key`; the codec does not read the underscore keys. +LEGACY_KEY = { + "id": 1, + "_schema": "s", + "_table": "t", + "_field": "f", + "_config": dj.config, +} + + +@pytest.mark.parametrize( + "context", + [None, {"schema": "s", "table": "t", "field": "f"}], + ids=["no context", "no config"], +) +def test_encode_needs_context(context: dict[str, Any] | None) -> None: + with pytest.raises(dj.DataJointError, match=r"DataJoint 2\.3\.4"): + PlotlyCodec().encode( + _figure(), key=LEGACY_KEY, context=context, store_name="store" + ) + + +def test_decode_needs_context() -> None: + stored = {"path": "f.plotly.json", "store": None, "size": 1, "traces": 0} + with pytest.raises(dj.DataJointError, match=r"missing: config"): + PlotlyCodec().decode(stored, key={"_config": dj.config}, context=None) + + # Through DataJoint, with a database and a file store diff --git a/uv.lock b/uv.lock index aa48027..af06af4 100644 --- a/uv.lock +++ b/uv.lock @@ -579,7 +579,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "datajoint", specifier = ">=2.3" }, + { name = "datajoint", specifier = ">=2.3.4" }, { name = "numpy", specifier = ">=2.0" }, { name = "plotly", marker = "extra == 'plotly'", specifier = ">=5" }, ]