From e22eaa5ae647a9f7f1cb09013b67c7751bcfd597 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 05:32:49 -0400 Subject: [PATCH 1/3] Ask policyengine.py to qualify each published data release After the data uploads and the tag is published, the release build sends a policyengine-uk-data-release repository dispatch to policyengine.py, whose certify-uk-release workflow qualifies the new release against the latest model before it can be promoted. The token comes from the org App and is scoped to policyengine.py alone. Without this, qualification waits for its two-hourly schedule. Workflow-only: no data or package change, so no changelog fragment (a fragment would bump the version and release data outside the batch). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 26 +++++ .../tests/test_release_dispatch.py | 105 ++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 policyengine_uk_data/tests/test_release_dispatch.py diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 87c2e406..72f5c58f 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -158,3 +158,29 @@ jobs: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - name: Publish a git tag run: ".github/publish-git-tag.sh" + # policyengine.py qualifies every new data release against the latest + # model before it can be promoted (certify-uk-release.yaml). Tell it now + # rather than leave it to its two-hourly schedule. This runs only after + # the data uploaded and the tag was published; a duplicate dispatch for + # a version already qualified is a no-op there. + - name: Generate a token for policyengine.py + id: qualification-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: PolicyEngine + repositories: policyengine.py + - name: Ask policyengine.py to qualify this release + env: + GH_TOKEN: ${{ steps.qualification-token.outputs.token }} + run: | + version=$(python3 .github/fetch_version.py) + if [ -z "$version" ] || [ "$version" = None ]; then + echo "::error::Could not read the released version from pyproject.toml." + exit 1 + fi + gh api --method POST repos/PolicyEngine/policyengine.py/dispatches \ + -f event_type=policyengine-uk-data-release \ + -f "client_payload[version]=${version}" + echo "Asked policyengine.py to qualify policyengine-uk-data ${version}." diff --git a/policyengine_uk_data/tests/test_release_dispatch.py b/policyengine_uk_data/tests/test_release_dispatch.py new file mode 100644 index 00000000..6eed70c3 --- /dev/null +++ b/policyengine_uk_data/tests/test_release_dispatch.py @@ -0,0 +1,105 @@ +"""The release build asks policyengine.py to qualify each published release. + +policyengine.py's certify-uk-release workflow qualifies a new data release +against the latest model before it can be promoted, triggered by a +``policyengine-uk-data-release`` repository dispatch whose payload names the +version. The dispatch must come only after the data uploaded and the tag was +published, carry exactly the released version, and use a token scoped to +policyengine.py alone. +""" + +import os +import subprocess +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +PUSH_WORKFLOW = ROOT / ".github/workflows/push.yaml" +TOKEN_STEP = "Generate a token for policyengine.py" +DISPATCH_STEP = "Ask policyengine.py to qualify this release" + + +def _steps(): + job = yaml.safe_load(PUSH_WORKFLOW.read_text())["jobs"]["test"] + return [step.get("name") for step in job["steps"]], { + step.get("name"): step for step in job["steps"] + } + + +def test_qualification_is_requested_only_after_the_release_is_published(): + order, by_name = _steps() + assert ( + order.index("Upload data") + < order.index("Publish a git tag") + < order.index(TOKEN_STEP) + < order.index(DISPATCH_STEP) + ) + # No condition: the default success() means every earlier step, the + # upload included, succeeded. + assert "if" not in by_name[TOKEN_STEP] + assert "if" not in by_name[DISPATCH_STEP] + assert not by_name[DISPATCH_STEP].get("continue-on-error", False) + + +def test_the_token_reaches_policyengine_py_only(): + _, by_name = _steps() + token = by_name[TOKEN_STEP] + assert token["uses"].startswith("actions/create-github-app-token@") + assert token["with"]["owner"] == "PolicyEngine" + assert token["with"]["repositories"] == "policyengine.py" + dispatch = by_name[DISPATCH_STEP] + assert dispatch["env"] == { + "GH_TOKEN": "${{ steps.qualification-token.outputs.token }}" + } + assert "${{" not in dispatch["run"] + + +def _run_dispatch(tmp_path, pyproject_version): + """Run the dispatch step's script with a fake gh that records its arguments.""" + + _, by_name = _steps() + repo = tmp_path / "repo" + (repo / ".github").mkdir(parents=True) + (repo / ".github/fetch_version.py").write_text( + (ROOT / ".github/fetch_version.py").read_text() + ) + (repo / "pyproject.toml").write_text( + f'[project]\nversion = "{pyproject_version}"\n' + if pyproject_version + else "[project]\n" + ) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + log = tmp_path / "gh.log" + (bin_dir / "gh").write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{log}"\n') + (bin_dir / "gh").chmod(0o755) + result = subprocess.run( + ["bash", "-e", "-c", by_name[DISPATCH_STEP]["run"]], + cwd=repo, + env={"PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}"}, + capture_output=True, + text=True, + ) + return result, log.read_text().splitlines() if log.exists() else None + + +def test_the_dispatch_names_the_released_version(tmp_path): + result, arguments = _run_dispatch(tmp_path, "1.59.0") + assert result.returncode == 0, result.stderr + assert arguments == [ + "api", + "--method", + "POST", + "repos/PolicyEngine/policyengine.py/dispatches", + "-f", + "event_type=policyengine-uk-data-release", + "-f", + "client_payload[version]=1.59.0", + ] + + +def test_no_dispatch_without_a_version(tmp_path): + result, arguments = _run_dispatch(tmp_path, None) + assert result.returncode != 0 + assert arguments is None From 054d026557a41d4d8fc4946ab784658a724e3a69 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 06:58:16 -0400 Subject: [PATCH 2/3] Dispatch only once the tag is on origin; scope the token to contents publish-git-tag.sh ignores a failed tag push, so the dispatch step now checks the release tag is on origin (git ls-remote) and fails without dispatching if it is not. The App token is restricted to permission-contents: write, all a repository dispatch needs. Tests cover the missing tag and the token's permissions. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/push.yaml | 8 ++++++ .../tests/test_release_dispatch.py | 26 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 72f5c58f..3b79ba4d 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -171,6 +171,8 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: PolicyEngine repositories: policyengine.py + # A repository dispatch needs contents: write and nothing else. + permission-contents: write - name: Ask policyengine.py to qualify this release env: GH_TOKEN: ${{ steps.qualification-token.outputs.token }} @@ -180,6 +182,12 @@ jobs: echo "::error::Could not read the released version from pyproject.toml." exit 1 fi + # publish-git-tag.sh ignores a failed tag push, so check the tag is + # really on origin before announcing the release. + if ! git ls-remote --exit-code --tags origin "refs/tags/${version}" > /dev/null; then + echo "::error::Tag ${version} is not on origin, so the release is not published; not asking policyengine.py to qualify it." + exit 1 + fi gh api --method POST repos/PolicyEngine/policyengine.py/dispatches \ -f event_type=policyengine-uk-data-release \ -f "client_payload[version]=${version}" diff --git a/policyengine_uk_data/tests/test_release_dispatch.py b/policyengine_uk_data/tests/test_release_dispatch.py index 6eed70c3..a8583a8d 100644 --- a/policyengine_uk_data/tests/test_release_dispatch.py +++ b/policyengine_uk_data/tests/test_release_dispatch.py @@ -48,6 +48,9 @@ def test_the_token_reaches_policyengine_py_only(): assert token["uses"].startswith("actions/create-github-app-token@") assert token["with"]["owner"] == "PolicyEngine" assert token["with"]["repositories"] == "policyengine.py" + # Only the permission a dispatch needs, not all the App's permissions. + assert token["with"]["permission-contents"] == "write" + assert not [key for key in token["with"] if key.startswith("permission-")][1:] dispatch = by_name[DISPATCH_STEP] assert dispatch["env"] == { "GH_TOKEN": "${{ steps.qualification-token.outputs.token }}" @@ -55,8 +58,11 @@ def test_the_token_reaches_policyengine_py_only(): assert "${{" not in dispatch["run"] -def _run_dispatch(tmp_path, pyproject_version): - """Run the dispatch step's script with a fake gh that records its arguments.""" +def _run_dispatch(tmp_path, pyproject_version, *, tag_on_origin=True): + """Run the dispatch step's script with a fake gh that records its arguments. + + A fake git answers ``ls-remote`` for the release tag as origin would. + """ _, by_name = _steps() repo = tmp_path / "repo" @@ -74,6 +80,13 @@ def _run_dispatch(tmp_path, pyproject_version): log = tmp_path / "gh.log" (bin_dir / "gh").write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{log}"\n') (bin_dir / "gh").chmod(0o755) + (bin_dir / "git").write_text( + "#!/usr/bin/env bash\n" + 'if [ "$1" = ls-remote ]; then exit ' + + ("0" if tag_on_origin else "2") + + "; fi\nexit 1\n" + ) + (bin_dir / "git").chmod(0o755) result = subprocess.run( ["bash", "-e", "-c", by_name[DISPATCH_STEP]["run"]], cwd=repo, @@ -103,3 +116,12 @@ def test_no_dispatch_without_a_version(tmp_path): result, arguments = _run_dispatch(tmp_path, None) assert result.returncode != 0 assert arguments is None + + +def test_no_dispatch_when_the_tag_did_not_reach_origin(tmp_path): + """publish-git-tag.sh swallows a failed push; the dispatch must not.""" + + result, arguments = _run_dispatch(tmp_path, "1.59.0", tag_on_origin=False) + assert result.returncode != 0 + assert "not on origin" in result.stdout + result.stderr + assert arguments is None From 1577a38d3ebd455eb9b2f2bedfe5784e696ac552 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 07:13:40 -0400 Subject: [PATCH 3/3] Check the tag lookup's exact arguments and fail on a rejected dispatch The fake git now answers ls-remote as git does (a missing ref exits 2 only under --exit-code), so dropping --exit-code or masking a failed gh api call fails the tests. Round-2 review finding. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_release_dispatch.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/policyengine_uk_data/tests/test_release_dispatch.py b/policyengine_uk_data/tests/test_release_dispatch.py index a8583a8d..e97277d6 100644 --- a/policyengine_uk_data/tests/test_release_dispatch.py +++ b/policyengine_uk_data/tests/test_release_dispatch.py @@ -58,10 +58,12 @@ def test_the_token_reaches_policyengine_py_only(): assert "${{" not in dispatch["run"] -def _run_dispatch(tmp_path, pyproject_version, *, tag_on_origin=True): +def _run_dispatch(tmp_path, pyproject_version, *, tag_on_origin=True, gh_fails=False): """Run the dispatch step's script with a fake gh that records its arguments. - A fake git answers ``ls-remote`` for the release tag as origin would. + A fake git answers ``ls-remote`` as git does: a missing ref exits 2 only + under ``--exit-code`` and otherwise exits 0 with no output, so the step + must ask for it. Its arguments are recorded in ``git.log``. """ _, by_name = _steps() @@ -78,13 +80,20 @@ def _run_dispatch(tmp_path, pyproject_version, *, tag_on_origin=True): bin_dir = tmp_path / "bin" bin_dir.mkdir() log = tmp_path / "gh.log" - (bin_dir / "gh").write_text(f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{log}"\n') + (bin_dir / "gh").write_text( + f'#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "{log}"\n' + + ("echo 'HTTP 403' >&2\nexit 1\n" if gh_fails else "") + ) (bin_dir / "gh").chmod(0o755) (bin_dir / "git").write_text( "#!/usr/bin/env bash\n" - 'if [ "$1" = ls-remote ]; then exit ' - + ("0" if tag_on_origin else "2") - + "; fi\nexit 1\n" + f'printf "%s\\n" "$@" > "{tmp_path / "git.log"}"\n' + 'if [ "$1" != ls-remote ]; then exit 1; fi\n' + + ( + 'printf "%s\\trefs/tags/%s\\n" 0123abcd "${@: -1}"\nexit 0\n' + if tag_on_origin + else 'for arg in "$@"; do [ "$arg" = --exit-code ] && exit 2; done\nexit 0\n' + ) ) (bin_dir / "git").chmod(0o755) result = subprocess.run( @@ -125,3 +134,26 @@ def test_no_dispatch_when_the_tag_did_not_reach_origin(tmp_path): assert result.returncode != 0 assert "not on origin" in result.stdout + result.stderr assert arguments is None + + +def test_the_tag_lookup_asks_origin_for_exactly_the_release_tag(tmp_path): + """Without --exit-code, git ls-remote succeeds when the tag is missing.""" + + result, _ = _run_dispatch(tmp_path, "1.59.0") + assert result.returncode == 0, result.stderr + assert (tmp_path / "git.log").read_text().splitlines() == [ + "ls-remote", + "--exit-code", + "--tags", + "origin", + "refs/tags/1.59.0", + ] + + +def test_a_failed_dispatch_fails_the_step(tmp_path): + """A rejected dispatch must not pass silently, or no qualification runs.""" + + result, arguments = _run_dispatch(tmp_path, "1.59.0", gh_fails=True) + assert result.returncode != 0 + assert arguments is not None + assert "Asked policyengine.py" not in result.stdout