Accumulate release notes in a version-less NEXT.md (#58)

.dev/tools/check-release-notes.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-"""Verify that every doc/release-notes/rel-X.Y.Z.md carries a correct verification section and changelog link.
+"""Verify that every doc/release-notes/rel-X.Y.Z.md carries a correct verification section and changelog link,
+and that the NEXT.md draft carries neither.
 
 The notes file is published verbatim as the GitHub release body (release.yml; plan/tickets/56/plan.md D1), so
 the artifact verification instructions and the changelog link exist only if the file contains them.  This checks,
@@ -12,6 +13,12 @@
     the GitHub Actions OIDC issuer;
   - there is a `**Full Changelog**:` compare link ending at this file's tag and starting at an earlier one.
 
+doc/release-notes/NEXT.md is the version-less draft that accumulates during a release cycle and is renamed to
+rel-<version>.md at the cut (#58).  Every part checked above names the release's tag, so in NEXT.md each one is a
+guess at a version not yet decided; there the check is inverted, and a verification heading, a `sigstore verify
+identity` command, a `--cert-identity`, or a Full Changelog line is an error.  Only the real thing counts: a
+heading or command at the start of a line, not the draft's own checklist mentioning them in prose.
+
 The identity check is the one that earns its keep.  The section is hand-written per release, usually by copying
 the previous one, and a stale tag in the identity makes `sigstore verify` reject every genuine artifact of the
 release ("Certificate's SANs do not match").  Readers would reasonably conclude the release is forged.  The
@@ -26,7 +33,8 @@
 Usage:
     python .dev/tools/check-release-notes.py [FILE ...]
 
-With no arguments, checks every doc/release-notes/rel-*.md.  Stdlib only.  Exits 0 if all pass, 1 otherwise.
+With no arguments, checks every doc/release-notes/rel-*.md, and NEXT.md if present.  Stdlib only.  Exits 0 if all
+pass, 1 otherwise.
 """
 
 from __future__ import annotations
@@ -37,6 +45,7 @@
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 NOTES_DIR = REPO_ROOT / "doc" / "release-notes"
+NEXT_NAME = "NEXT.md"
 
 REPOSITORY = "osprey-dcs/dp-python-lib"
 OIDC_ISSUER = "https://token.actions.githubusercontent.com"
@@ -47,6 +56,8 @@
 VERIFY_RE = re.compile(r"\bsigstore\s+verify\s+identity\b(?:[^\n]*\\\n)*[^\n]*")
 IDENTITY_RE = re.compile(r"--cert-identity[ =]\"?([^\"\s]+)\"?")
 ISSUER_RE = re.compile(r"--cert-oidc-issuer[ =]\"?([^\"\s]+)\"?")
+# A command as written in a code block, rather than named in prose.
+VERIFY_COMMAND_RE = re.compile(r"^[ \t]*sigstore\s+verify\s+identity\b", re.MULTILINE)
 CHANGELOG_RE = re.compile(r"^\*\*Full Changelog\*\*:\s*(\S+)\s*$", re.MULTILINE)
 COMPARE_RE = re.compile(
     rf"^https://github\.com/{re.escape(REPOSITORY)}/compare/(rel-\d+\.\d+\.\d+)\.\.\.(rel-\d+\.\d+\.\d+)$"
@@ -137,11 +148,31 @@ def check_text(name: str, tag: str, text: str) -> list[str]:
     return problems
 
 
+def check_next_text(name: str, text: str) -> list[str]:
+    """Returns one message per tag-bearing part found in the NEXT.md draft `text`; empty when it has none."""
+    problems: list[str] = []
+    forbidden = [
+        (HEADING_RE, "a '## Verifying these artifacts' section"),
+        (VERIFY_COMMAND_RE, "a 'sigstore verify identity' command"),
+        (IDENTITY_RE, "a --cert-identity"),
+        (CHANGELOG_RE, "a '**Full Changelog**' line"),
+    ]
+    for pattern, description in forbidden:
+        if pattern.search(text):
+            problems.append(
+                f"{name}: contains {description}, which names the release's tag; NEXT.md names no version, "
+                "so add it when the file is renamed at the cut"
+            )
+    return problems
+
+
 def check_file(path: Path) -> list[str]:
     """Returns one message per problem found in `path`; empty when it passes."""
+    if path.name == NEXT_NAME:
+        return check_next_text(str(path), path.read_text(encoding="utf-8"))
     tag = path.stem
     if path.suffix != ".md" or not STEM_RE.match(tag):
-        return [f"{path}: name must be rel-X.Y.Z.md (release.yml looks the notes up by tag)"]
+        return [f"{path}: name must be rel-X.Y.Z.md or {NEXT_NAME} (release.yml looks the notes up by tag)"]
     return check_text(str(path), tag, path.read_text(encoding="utf-8"))
 
 
@@ -189,6 +220,27 @@ def self_test() -> list[str]:
     for description, notes in bad_cases.items():
         if not check_text(f"<{description}>", "rel-2.1.0", notes):
             failures.append(f"{description} was accepted")
+
+    # The draft's own checklist names each forbidden part in prose; that must not trip the rule.
+    next_draft = """# Release Notes — next release (unreleased)
+
+4. **Add the `## Verifying these artifacts` section**: `sigstore verify identity` over all three files, with
+   `--cert-identity` ending `release.yml@refs/tags/rel-<version>`.
+5. **End with the Full Changelog line**:
+   `**Full Changelog**: https://github.com/osprey-dcs/dp-python-lib/compare/rel-<previous>...rel-<version>`.
+"""
+    good_next = check_next_text("<good NEXT.md>", next_draft)
+    if good_next:
+        failures.append("a correct NEXT.md was rejected:\n      " + "\n      ".join(good_next))
+    for description, notes in {
+        "a NEXT.md with a verification section": next_draft + "\n## Verifying these artifacts\n",
+        "a NEXT.md with a verify command": next_draft + "\n```bash\nsigstore verify identity \\\n```\n",
+        "a NEXT.md with a signing identity": next_draft + f'\n  --cert-identity "{expected_identity("rel-2.1.0")}"\n',
+        "a NEXT.md with a Full Changelog line": next_draft
+        + f"\n**Full Changelog**: https://github.com/{REPOSITORY}/compare/rel-2.0.0...rel-2.1.0\n",
+    }.items():
+        if not check_next_text(f"<{description}>", notes):
+            failures.append(f"{description} was accepted")
     return failures
 
 
@@ -200,7 +252,12 @@ def main(argv: list[str]) -> int:
             print(f"  {failure}")
         return 1
 
-    paths = [Path(arg) for arg in argv] if argv else sorted(NOTES_DIR.glob("rel-*.md"))
+    if argv:
+        paths = [Path(arg) for arg in argv]
+    else:
+        paths = sorted(NOTES_DIR.glob("rel-*.md"))
+        if (NOTES_DIR / NEXT_NAME).is_file():
+            paths.append(NOTES_DIR / NEXT_NAME)
     if not paths:
         print(f"FAIL: no release notes found under {NOTES_DIR}")
         return 1

.github/workflows/ci.yml

@@ -98,7 +98,9 @@ jobs:
       - name: Check release notes
         # Each doc/release-notes/rel-X.Y.Z.md is published verbatim as its release body, so it
         # must carry a verification section whose signing identity names its own tag.  Checked
-        # here so a mistake fails the notes PR, not the tag push.
+        # here so a mistake fails the notes PR, not the tag push.  doc/release-notes/NEXT.md, the
+        # version-less draft renamed at the cut, is checked the other way round: it must carry none
+        # of those tag-bearing parts, since any it has guesses a version not yet decided.
         run: python .dev/tools/check-release-notes.py
 
   build:

CLAUDE.md

@@ -109,6 +109,17 @@ a note by issue ticket rather than by PR, since a ticket often spans several PRs
 a breaking release with an "Upgrading from <previous>" checklist that separates silent
 behavior changes from outright errors.
 
+**During a cycle, notes accumulate in `doc/release-notes/NEXT.md`** (#58; `plan/tickets/58/plan.md`),
+the convention dp-grpc adopted in osprey-dcs/dp-grpc#156.  A ticket that changes anything a user of
+the library or its release artifacts would notice adds its section to `NEXT.md` **in the same PR**,
+while the reasoning is fresh and in front of the reviewer.  `NEXT.md` names no upcoming version, in
+its filename or its prose: the next version (1.17.0 or 2.0.0, say) is decided at the cut, and a file
+committed under a guessed `rel-<version>.md` is stranded and fails the release-notes check on the
+tag that does ship.  The cut is `git mv doc/release-notes/NEXT.md doc/release-notes/rel-<version>.md`
+plus the "Cutting the release" checklist at the bottom of `NEXT.md` itself, which ends by starting a
+fresh `NEXT.md`.  `release.yml` needs no change for this: it resolves the notes path strictly from
+the tag, so `NEXT.md` can never be published.
+
 **The notes file is the release body, verbatim** (#56; `plan/tickets/56/plan.md`), the same
 arrangement as the Java repos.  `release.yml` copies it to `dist/RELEASE_NOTES.md` (the publish job
 never checks out the repo, so it travels with the `dist/` upload) and passes that as `body_path`;
@@ -131,6 +142,12 @@ file, so `<prev>` is checked only for being earlier.  Each run starts with a sel
 the rules known-bad notes, so a rule that stops matching fails loudly instead of passing everything.
 It runs in CI's quality job and again at tag time.
 
+The same checker inverts the rules for `NEXT.md`: a verification heading, a `sigstore verify identity`
+command, a `--cert-identity`, or a Full Changelog line there is an error, since each names the tag and
+so, in the draft, guesses it.  Only real ones count -- a heading or command at the start of a line, an
+identity with a value, a changelog line at the start of a line -- because the draft's own checklist
+names all four in prose, and the self-test holds that prose to passing.
+
 **Never edit a release page by hand.**  Fix the notes file by PR, then republish with
 `gh release edit rel-X.Y.Z --notes-file doc/release-notes/rel-X.Y.Z.md`.  With the file as the
 whole body that is lossless; when the workflow appended a verification section, exactly this

doc/release-notes/NEXT.md

@@ -0,0 +1,117 @@
+# Release Notes — next release (unreleased)
+
+**This is the working draft for the next release. It is not a release note yet.**
+
+Sections accumulate here as tickets land, so the content is written while it is fresh and gets
+reviewed in the PR that causes it.  A ticket that changes anything a user of the library, or of its
+release artifacts, would notice adds its section here in that same PR.  At release time this file is
+renamed to `doc/release-notes/rel-<version>.md` and finished — see **Cutting the release** at the
+bottom.
+
+**The version of the upcoming release is deliberately not named anywhere in this file**, in its
+filename or in its prose.  `release.yml` resolves the notes path strictly from the tag
+(`doc/release-notes/${GITHUB_REF_NAME}.md`), so a file committed under a guessed version is both
+stranded and a failed release-notes check on the tag that does ship.  Past versions are named
+freely where they are the point — "since 1.16.0" is a durable fact about what shipped, not a guess
+about what is about to.  `.dev/tools/check-release-notes.py` enforces the parts that would carry the
+new tag: this file may not contain a verification section, a `sigstore verify identity` command, a
+signing identity, or a Full Changelog line.  Those are written at the cut.  Naming them in prose
+is fine, as the checklist below does, but put the name in backticks: the checker treats an unquoted
+`--cert-identity` followed by a word, or a line beginning with the verify command, as the real thing.
+
+Nothing here should assert what *else* the release contains, either: that is knowable only once
+the release is cut, and a stale claim in a file that already looks finished is not something the
+person cutting the release has any reason to re-read.
+
+## Contents
+
+- [Release pages are the notes file, verbatim (#56)](#release-pages-are-the-notes-file-verbatim-issue-56)
+- [Type checking in CI (#30)](#type-checking-in-ci-issue-30)
+- [Cutting the release](#cutting-the-release)
+
+---
+
+## Release pages are the notes file, verbatim (Issue #56)
+
+The GitHub release page is now exactly `doc/release-notes/rel-<version>.md`, with nothing added by
+the release workflow.  Previously the workflow appended artifact verification instructions and
+GitHub's generated commit list after the notes.  That arrangement broke the first time anyone
+republished a page from its notes file: rel-1.16.0's page silently lost its verification
+instructions that way.  With the file as the whole page, republishing is lossless.
+
+What changes for someone downloading a release:
+
+- **Signature verification covers all three files.**  The instructions previously verified only the
+  wheel, although the sdist and `SHA256SUMS` have always been signed too.  They now verify all three
+  in one call, and the new [`README.env`](https://github.com/osprey-dcs/dp-python-lib/blob/main/README.env)
+  is the full reference for what each artifact is and how to check it.  rel-1.16.0's page has been
+  republished with the corrected instructions.
+- **The commit list is replaced by a Full Changelog link**, a compare view between the two release
+  tags.  The notes themselves are organized by ticket and link their PRs.
+
+The release workflow checks that the published page matches the notes file after every release,
+and CI checks every notes file's verification section for the right signing identity, since a stale
+tag copied from the previous release makes `sigstore verify` reject every genuine artifact.
+
+## Type checking in CI (Issue #30)
+
+`mypy src/` is now clean and runs in CI on every PR, so type errors in the library fail review
+rather than reaching a release.  The generated gRPC stubs are excluded until dp-grpc ships typed
+ones ([osprey-dcs/dp-grpc#158](https://github.com/osprey-dcs/dp-grpc/issues/158)).  Nothing about
+the library's behavior changes.
+
+Two annotations became more accurate along the way:
+
+- **`MldpClient.annotation` and `MldpClient.query` are annotated `X | None`**, which is what they
+  have always been: they are `None` when the client is given only an ingestion channel.  The package
+  does not yet ship a `py.typed` marker, so your own mypy runs are unaffected.  An editor that infers
+  types from library source may now point out that they can be `None`; narrow once with
+  `assert client.annotation is not None` if yours does.
+- **`timestamp_list()` accepts any sequence** of timestamps (a tuple, say), not only a `list`.
+
+## Installing
+
+```bash
+pip install dp_python_lib-*.whl
+```
+
+---
+
+## Cutting the release
+
+When the version is known and the release is being cut:
+
+1. **`git mv doc/release-notes/NEXT.md doc/release-notes/rel-<version>.md`.**  The filename must
+   match the tag exactly; `release.yml` fails the run before the build if it does not.
+2. **Retitle** the H1 to `# dp-python-lib <version> Release Notes` and replace this file's preamble
+   with a "Changes since rel-<previous>" summary — written now, when the full contents of the
+   release are actually known.  If the stubs were resynced, link dp-grpc's notes for the same
+   release, as rel-1.16.0's opening does.
+3. **Decide whether the release is breaking**, and say so in the opening if it is.  A breaking
+   release gets an **"Upgrading from <previous>"** section as the first section after Contents,
+   folding in the per-ticket upgrade items above.  Call out silent behavior changes separately from
+   outright errors, per CLAUDE.md: a change that alters results without raising is the one a reader
+   most needs up front.  Python has no compile step, so "outright errors" here means ones raised at
+   import or call time.
+4. **Add the `## Verifying these artifacts` section** immediately above `## Installing`, copied from the previous release's notes with
+   the tag changed: `sha256sum -c SHA256SUMS`, then `sigstore verify identity` over the wheel,
+   sdist, and `SHA256SUMS` with `--cert-identity` ending `release.yml@refs/tags/rel-<version>`, and
+   the pointer to `README.env`.
+5. **End with the Full Changelog line**, after `## Installing`:
+   `**Full Changelog**: https://github.com/osprey-dcs/dp-python-lib/compare/rel-<previous>...rel-<version>`.
+6. **Repoint `blob/main/...` links to `blob/rel-<version>/...`.**  This file is published as the
+   release body via `body_path`, and relative links do not survive that lift — they resolve against
+   the repo root, not `doc/release-notes/`, and 404.  Links here are already absolute for that
+   reason, but one pinned to `main` drifts as the repo moves on; pinned to the tag it keeps
+   describing the content this release actually shipped.  (`README.env` did not exist at 1.16.0,
+   so a link to it from older notes stays on `main`.)
+7. **Delete this "Cutting the release" section** and update Contents.
+8. **Run `python .dev/tools/check-release-notes.py`**, which CI also runs on the PR.  It fails on a
+   missing verification section, a stale tag in the identity or the changelog link, or a verify
+   command that skips a file.
+9. **Start a fresh `NEXT.md`** for the following cycle.  Steps 1, 2, and 7 have moved, rewritten, and
+   deleted the text it needs, so recover it from `main`:
+   `git show main:doc/release-notes/NEXT.md > doc/release-notes/NEXT.md`, then delete every ticket
+   section and empty Contents down to the "Cutting the release" entry.  Keep the preamble,
+   `## Installing`, and this checklist.
+10. **Merge, then push the `rel-<version>` tag.**  The notes must be on the tagged commit.