diff --git a/.github/workflows/refresh-upstream.yml b/.github/workflows/refresh-upstream.yml new file mode 100644 index 0000000..5096664 --- /dev/null +++ b/.github/workflows/refresh-upstream.yml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +name: Refresh upstream workflows + +on: + workflow_dispatch: + schedule: + - cron: '17 3 * * 0' + +permissions: + contents: read + +jobs: + refresh: + name: Refresh pinned upstream sources + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Refresh upstream pins + env: + GITHUB_TOKEN: ${{ github.token }} + run: python3 scripts/sync_upstream.py refresh upstream/sources.json + + - name: Verify refreshed sources + run: | + python3 scripts/sync_upstream.py check upstream/sources.json + python3 -m unittest discover -s tests -p 'test_*.py' + + - name: Create update pull request + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ secrets.WORKFLOW_UPDATE_TOKEN }} + commit-message: 'chore: refresh upstream workflow pins' + committer: GitHub + author: github-workflows bot + signoff: true + branch: automated/refresh-upstream-workflows + delete-branch: true + title: 'chore: refresh upstream workflow pins' + body: | + Automated refresh of tracked upstream workflow sources. + + The committed source URLs remain pinned to immutable commit SHAs and + SHA-256 hashes. Review upstream changes and any downstream patches + before merging. + labels: dependencies + add-paths: | + upstream/sources.json + upstream/vendor/** diff --git a/docs/upstream-workflows.md b/docs/upstream-workflows.md index 802609d..3bdd886 100644 --- a/docs/upstream-workflows.md +++ b/docs/upstream-workflows.md @@ -10,9 +10,14 @@ Upstream files are declared in `upstream/sources.json`. Each entry contains: - `name`: stable local identifier; +- `repository`, `ref` and `path`: optional tracking metadata used only to discover newer upstream revisions; - `url`: raw file URL pinned to an immutable upstream commit; - `sha256`: expected SHA-256 of the downloaded bytes; -- `destination`: repository-relative generated destination. +- `destination`: repository-relative vendored destination. + +The tracking ref can be mutable. The effective source cannot: after refresh, the +manifest is rewritten to a full commit SHA and content hash before the vendored +file is accepted. Example: @@ -21,9 +26,12 @@ Example: "sources": [ { "name": "example", + "repository": "example/project", + "ref": "main", + "path": ".github/workflows/example.yml", "url": "https://raw.githubusercontent.com/example/project//.github/workflows/example.yml", "sha256": "<64 lowercase hex characters>", - "destination": "templates/example.yml" + "destination": "upstream/vendor/example/example.yml" } ] } @@ -31,20 +39,36 @@ Example: ## Commands -Synchronize declared sources: +Synchronize declared immutable sources: ```bash python3 scripts/sync_upstream.py sync upstream/sources.json ``` -Verify committed generated files without modifying them: +Verify committed vendored files without modifying them: ```bash python3 scripts/sync_upstream.py check upstream/sources.json ``` -Both commands verify the source hash before accepting content. +Resolve tracked refs to their latest commit, recompute SHA-256 and update the +vendored files: + +```bash +python3 scripts/sync_upstream.py refresh upstream/sources.json +``` + +The scheduled `refresh-upstream.yml` workflow runs this refresh weekly, validates +the result and opens a pull request when upstream changed. + +A dedicated `WORKFLOW_UPDATE_TOKEN` secret is required for pull-request creation. +Using only the workflow's `GITHUB_TOKEN` would prevent the resulting pull request +from triggering the normal CI workflows. The refresh itself only uses the +read-only `GITHUB_TOKEN` to resolve public upstream commits. + +Both `sync` and `check` verify the recorded source hash before accepting +content. `refresh` only records bytes fetched from the exact commit it resolved. -Patch application will be introduced with the first real upstream template so -the patch interface is designed against an actual workflow rather than a -hypothetical format. +Patch application is intentionally a separate layer: upstream bytes remain +verbatim under `upstream/vendor/`, while downstream adaptations should be stored +as reviewable patches and rendered into generated templates. diff --git a/scripts/sync_upstream.py b/scripts/sync_upstream.py index 3ace132..a09546e 100755 --- a/scripts/sync_upstream.py +++ b/scripts/sync_upstream.py @@ -7,9 +7,10 @@ import argparse import hashlib import json +import os from dataclasses import dataclass from pathlib import Path -from urllib.parse import urlparse +from urllib.parse import quote, urlparse from urllib.request import Request, urlopen @@ -19,6 +20,9 @@ class Source: url: str sha256: str destination: Path + repository: str | None = None + ref: str | None = None + path: str | None = None def load_sources(manifest_path: Path) -> list[Source]: @@ -46,12 +50,26 @@ def load_sources(manifest_path: Path) -> list[Source]: raise ValueError(f"sources[{index}].sha256 must be 64 lowercase hex characters") _validate_immutable_url(url, f"sources[{index}].url") + repository = _optional_string(raw.get("repository"), f"sources[{index}].repository") + ref = _optional_string(raw.get("ref"), f"sources[{index}].ref") + path = _optional_string(raw.get("path"), f"sources[{index}].path") + tracking = (repository, ref, path) + if any(value is not None for value in tracking) and not all( + value is not None for value in tracking + ): + raise ValueError( + f"sources[{index}] must define repository, ref and path together" + ) + sources.append( Source( name=name, url=url, sha256=digest, destination=Path(destination), + repository=repository, + ref=ref, + path=path, ) ) @@ -59,10 +77,7 @@ def load_sources(manifest_path: Path) -> list[Source]: def fetch(source: Source) -> bytes: - request = Request(source.url, headers={"User-Agent": "github-workflows-sync"}) - with urlopen(request, timeout=30) as response: - content = response.read() - + content = _download(source.url) actual = hashlib.sha256(content).hexdigest() if actual != source.sha256: raise ValueError( @@ -91,6 +106,78 @@ def check(sources: list[Source], root: Path) -> None: raise ValueError("generated templates are out of date: " + ", ".join(drift)) +def refresh(manifest_path: Path, root: Path, token: str | None = None) -> None: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + raw_sources = payload.get("sources") + if not isinstance(raw_sources, list): + raise ValueError("manifest.sources must be an array") + + # Validate the current manifest before mutating it. + load_sources(manifest_path) + + for index, raw in enumerate(raw_sources): + if not isinstance(raw, dict): + raise ValueError(f"manifest.sources[{index}] must be an object") + + repository = raw.get("repository") + ref = raw.get("ref") + path = raw.get("path") + if not all(isinstance(value, str) and value for value in (repository, ref, path)): + continue + + commit = _latest_commit(repository, ref, path, token) + url = f"https://raw.githubusercontent.com/{repository}/{commit}/{path}" + content = _download(url) + digest = hashlib.sha256(content).hexdigest() + + raw["url"] = url + raw["sha256"] = digest + + destination = _safe_destination(root, Path(str(raw["destination"]))) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(content) + + manifest_path.write_text( + json.dumps(payload, indent=2, sort_keys=False) + "\n", + encoding="utf-8", + ) + + +def _latest_commit(repository: str, ref: str, path: str, token: str | None) -> str: + url = ( + f"https://api.github.com/repos/{repository}/commits" + f"?sha={quote(ref, safe='')}&path={quote(path, safe='')}&per_page=1" + ) + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "github-workflows-sync", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + headers["Authorization"] = f"Bearer {token}" + + request = Request(url, headers=headers) + with urlopen(request, timeout=30) as response: + payload = json.load(response) + + if not isinstance(payload, list) or not payload: + raise ValueError( + f"cannot resolve latest commit for {repository}:{ref}:{path}" + ) + commit = payload[0].get("sha") + if not isinstance(commit, str) or len(commit) != 40: + raise ValueError( + f"invalid commit returned for {repository}:{ref}:{path}" + ) + return commit + + +def _download(url: str) -> bytes: + request = Request(url, headers={"User-Agent": "github-workflows-sync"}) + with urlopen(request, timeout=30) as response: + return response.read() + + def _validate_immutable_url(url: str, path: str) -> None: parsed = urlparse(url) if parsed.scheme != "https": @@ -125,20 +212,35 @@ def _non_empty_string(value: object, path: str) -> str: return value +def _optional_string(value: object, path: str) -> str | None: + if value is None: + return None + if not isinstance(value, str) or not value: + raise ValueError(f"{path} must be a non-empty string when defined") + return value + + def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("command", choices=("sync", "check")) + parser.add_argument("command", choices=("sync", "check", "refresh")) parser.add_argument("manifest", type=Path) args = parser.parse_args() root = Path.cwd() - sources = load_sources(args.manifest) try: - if args.command == "sync": - sync(sources, root) + if args.command == "refresh": + refresh( + args.manifest, + root, + token=os.environ.get("GITHUB_TOKEN"), + ) else: - check(sources, root) + sources = load_sources(args.manifest) + if args.command == "sync": + sync(sources, root) + else: + check(sources, root) except ValueError as error: parser.error(str(error)) diff --git a/tests/test_sync_upstream.py b/tests/test_sync_upstream.py index cc47bc4..93d5e49 100644 --- a/tests/test_sync_upstream.py +++ b/tests/test_sync_upstream.py @@ -8,7 +8,7 @@ from pathlib import Path from unittest.mock import patch -from scripts.sync_upstream import Source, check, load_sources, sync +from scripts.sync_upstream import Source, check, load_sources, refresh, sync class SyncUpstreamTest(unittest.TestCase): @@ -21,6 +21,9 @@ def test_loads_valid_manifest(self) -> None: "sources": [ { "name": "workflow", + "repository": "example/project", + "ref": "master", + "path": "workflow.yml", "url": "https://raw.githubusercontent.com/example/project/0123456789012345678901234567890123456789/workflow.yml", "sha256": "a" * 64, "destination": "templates/workflow.yml", @@ -33,8 +36,36 @@ def test_loads_valid_manifest(self) -> None: [source] = load_sources(manifest) self.assertEqual(source.name, "workflow") + self.assertEqual(source.repository, "example/project") + self.assertEqual(source.ref, "master") + self.assertEqual(source.path, "workflow.yml") self.assertEqual(source.destination, Path("templates/workflow.yml")) + def test_rejects_partial_tracking_metadata(self) -> None: + with tempfile.TemporaryDirectory() as directory: + manifest = Path(directory) / "sources.json" + manifest.write_text( + json.dumps( + { + "sources": [ + { + "name": "workflow", + "repository": "example/project", + "url": "https://raw.githubusercontent.com/example/project/0123456789012345678901234567890123456789/workflow.yml", + "sha256": "a" * 64, + "destination": "templates/workflow.yml", + } + ] + } + ), + encoding="utf-8", + ) + + with self.assertRaisesRegex( + ValueError, "must define repository, ref and path together" + ): + load_sources(manifest) + def test_rejects_invalid_hash(self) -> None: with tempfile.TemporaryDirectory() as directory: manifest = Path(directory) / "sources.json" @@ -116,6 +147,59 @@ def test_check_detects_drift(self) -> None: with self.assertRaisesRegex(ValueError, "out of date"): check([source], root) + @patch("scripts.sync_upstream._download") + @patch("scripts.sync_upstream._latest_commit") + def test_refresh_updates_pin_hash_and_vendor_copy( + self, latest_commit, download + ) -> None: + latest_commit.return_value = "1" * 40 + content = b"name: Updated\n" + download.return_value = content + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest = root / "sources.json" + manifest.write_text( + json.dumps( + { + "sources": [ + { + "name": "workflow", + "repository": "example/project", + "ref": "master", + "path": "workflow.yml", + "url": "https://raw.githubusercontent.com/example/project/0123456789012345678901234567890123456789/workflow.yml", + "sha256": "a" * 64, + "destination": "templates/workflow.yml", + } + ] + } + ), + encoding="utf-8", + ) + + refresh(manifest, root, token="token") + + payload = json.loads(manifest.read_text(encoding="utf-8")) + [source] = payload["sources"] + self.assertEqual( + source["url"], + "https://raw.githubusercontent.com/example/project/" + + "1" * 40 + + "/workflow.yml", + ) + self.assertEqual( + source["sha256"], + hashlib.sha256(content).hexdigest(), + ) + self.assertEqual( + (root / "templates/workflow.yml").read_bytes(), + content, + ) + latest_commit.assert_called_once_with( + "example/project", "master", "workflow.yml", "token" + ) + def test_rejects_destination_escape(self) -> None: content = b"name: Example\n" source = Source( diff --git a/upstream/sources.json b/upstream/sources.json index db95083..1cdf98d 100644 --- a/upstream/sources.json +++ b/upstream/sources.json @@ -2,6 +2,9 @@ "sources": [ { "name": "nextcloud-appstore-build-publish", + "repository": "nextcloud/.github", + "ref": "master", + "path": "workflow-templates/appstore-build-publish.yml", "url": "https://raw.githubusercontent.com/nextcloud/.github/cf6248d5ef28a328cde764daf80bc5fae4705ade/workflow-templates/appstore-build-publish.yml", "sha256": "4710d78c576abd1c875d799770f67262988119404d58d379656f5932872fcba8", "destination": "upstream/vendor/nextcloud/appstore-build-publish.yml"