feat: automate pinned upstream workflow refresh

.github/workflows/refresh-upstream.yml

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: Refresh upstream workflows
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '17 3 * * 0'
+
+permissions:
+  contents: read
+
+jobs:
+  refresh:
+    name: Refresh pinned upstream sources
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Refresh upstream pins
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: python3 scripts/sync_upstream.py refresh upstream/sources.json
+
+      - name: Verify refreshed sources
+        run: |
+          python3 scripts/sync_upstream.py check upstream/sources.json
+          python3 -m unittest discover -s tests -p 'test_*.py'
+
+      - name: Create update pull request
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
+        with:
+          token: ${{ secrets.WORKFLOW_UPDATE_TOKEN }}
+          commit-message: 'chore: refresh upstream workflow pins'
+          committer: GitHub <noreply@github.com>
+          author: github-workflows bot <noreply@github.com>
+          signoff: true
+          branch: automated/refresh-upstream-workflows
+          delete-branch: true
+          title: 'chore: refresh upstream workflow pins'
+          body: |
+            Automated refresh of tracked upstream workflow sources.
+
+            The committed source URLs remain pinned to immutable commit SHAs and
+            SHA-256 hashes. Review upstream changes and any downstream patches
+            before merging.
+          labels: dependencies
+          add-paths: |
+            upstream/sources.json
+            upstream/vendor/**

docs/upstream-workflows.md

@@ -10,9 +10,14 @@ Upstream files are declared in `upstream/sources.json`.
 Each entry contains:
 
 - `name`: stable local identifier;
+- `repository`, `ref` and `path`: optional tracking metadata used only to discover newer upstream revisions;
 - `url`: raw file URL pinned to an immutable upstream commit;
 - `sha256`: expected SHA-256 of the downloaded bytes;
-- `destination`: repository-relative generated destination.
+- `destination`: repository-relative vendored destination.
+
+The tracking ref can be mutable. The effective source cannot: after refresh, the
+manifest is rewritten to a full commit SHA and content hash before the vendored
+file is accepted.
 
 Example:
 
@@ -21,30 +26,49 @@ Example:
   "sources": [
     {
       "name": "example",
+      "repository": "example/project",
+      "ref": "main",
+      "path": ".github/workflows/example.yml",
       "url": "https://raw.githubusercontent.com/example/project/<commit>/.github/workflows/example.yml",
       "sha256": "<64 lowercase hex characters>",
-      "destination": "templates/example.yml"
+      "destination": "upstream/vendor/example/example.yml"
     }
   ]
 }
 ```
 
 ## Commands
 
-Synchronize declared sources:
+Synchronize declared immutable sources:
 
 ```bash
 python3 scripts/sync_upstream.py sync upstream/sources.json
 ```
 
-Verify committed generated files without modifying them:
+Verify committed vendored files without modifying them:
 
 ```bash
 python3 scripts/sync_upstream.py check upstream/sources.json
 ```
 
-Both commands verify the source hash before accepting content.
+Resolve tracked refs to their latest commit, recompute SHA-256 and update the
+vendored files:
+
+```bash
+python3 scripts/sync_upstream.py refresh upstream/sources.json
+```
+
+The scheduled `refresh-upstream.yml` workflow runs this refresh weekly, validates
+the result and opens a pull request when upstream changed.
+
+A dedicated `WORKFLOW_UPDATE_TOKEN` secret is required for pull-request creation.
+Using only the workflow's `GITHUB_TOKEN` would prevent the resulting pull request
+from triggering the normal CI workflows. The refresh itself only uses the
+read-only `GITHUB_TOKEN` to resolve public upstream commits.
+
+Both `sync` and `check` verify the recorded source hash before accepting
+content. `refresh` only records bytes fetched from the exact commit it resolved.
 
-Patch application will be introduced with the first real upstream template so
-the patch interface is designed against an actual workflow rather than a
-hypothetical format.
+Patch application is intentionally a separate layer: upstream bytes remain
+verbatim under `upstream/vendor/`, while downstream adaptations should be stored
+as reviewable patches and rendered into generated templates.

scripts/sync_upstream.py

@@ -7,9 +7,10 @@
 import argparse
 import hashlib
 import json
+import os
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
+from urllib.parse import quote, urlparse
 from urllib.request import Request, urlopen
 
 
@@ -19,6 +20,9 @@ class Source:
     url: str
     sha256: str
     destination: Path
+    repository: str | None = None
+    ref: str | None = None
+    path: str | None = None
 
 
 def load_sources(manifest_path: Path) -> list[Source]:
@@ -46,23 +50,34 @@ def load_sources(manifest_path: Path) -> list[Source]:
             raise ValueError(f"sources[{index}].sha256 must be 64 lowercase hex characters")
         _validate_immutable_url(url, f"sources[{index}].url")
 
+        repository = _optional_string(raw.get("repository"), f"sources[{index}].repository")
+        ref = _optional_string(raw.get("ref"), f"sources[{index}].ref")
+        path = _optional_string(raw.get("path"), f"sources[{index}].path")
+        tracking = (repository, ref, path)
+        if any(value is not None for value in tracking) and not all(
+            value is not None for value in tracking
+        ):
+            raise ValueError(
+                f"sources[{index}] must define repository, ref and path together"
+            )
+
         sources.append(
             Source(
                 name=name,
                 url=url,
                 sha256=digest,
                 destination=Path(destination),
+                repository=repository,
+                ref=ref,
+                path=path,
             )
         )
 
     return sources
 
 
 def fetch(source: Source) -> bytes:
-    request = Request(source.url, headers={"User-Agent": "github-workflows-sync"})
-    with urlopen(request, timeout=30) as response:
-        content = response.read()
-
+    content = _download(source.url)
     actual = hashlib.sha256(content).hexdigest()
     if actual != source.sha256:
         raise ValueError(
@@ -91,6 +106,78 @@ def check(sources: list[Source], root: Path) -> None:
         raise ValueError("generated templates are out of date: " + ", ".join(drift))
 
 
+def refresh(manifest_path: Path, root: Path, token: str | None = None) -> None:
+    payload = json.loads(manifest_path.read_text(encoding="utf-8"))
+    raw_sources = payload.get("sources")
+    if not isinstance(raw_sources, list):
+        raise ValueError("manifest.sources must be an array")
+
+    # Validate the current manifest before mutating it.
+    load_sources(manifest_path)
+
+    for index, raw in enumerate(raw_sources):
+        if not isinstance(raw, dict):
+            raise ValueError(f"manifest.sources[{index}] must be an object")
+
+        repository = raw.get("repository")
+        ref = raw.get("ref")
+        path = raw.get("path")
+        if not all(isinstance(value, str) and value for value in (repository, ref, path)):
+            continue
+
+        commit = _latest_commit(repository, ref, path, token)
+        url = f"https://raw.githubusercontent.com/{repository}/{commit}/{path}"
+        content = _download(url)
+        digest = hashlib.sha256(content).hexdigest()
+
+        raw["url"] = url
+        raw["sha256"] = digest
+
+        destination = _safe_destination(root, Path(str(raw["destination"])))
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_bytes(content)
+
+    manifest_path.write_text(
+        json.dumps(payload, indent=2, sort_keys=False) + "\n",
+        encoding="utf-8",
+    )
+
+
+def _latest_commit(repository: str, ref: str, path: str, token: str | None) -> str:
+    url = (
+        f"https://api.github.com/repos/{repository}/commits"
+        f"?sha={quote(ref, safe='')}&path={quote(path, safe='')}&per_page=1"
+    )
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "github-workflows-sync",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    request = Request(url, headers=headers)
+    with urlopen(request, timeout=30) as response:
+        payload = json.load(response)
+
+    if not isinstance(payload, list) or not payload:
+        raise ValueError(
+            f"cannot resolve latest commit for {repository}:{ref}:{path}"
+        )
+    commit = payload[0].get("sha")
+    if not isinstance(commit, str) or len(commit) != 40:
+        raise ValueError(
+            f"invalid commit returned for {repository}:{ref}:{path}"
+        )
+    return commit
+
+
+def _download(url: str) -> bytes:
+    request = Request(url, headers={"User-Agent": "github-workflows-sync"})
+    with urlopen(request, timeout=30) as response:
+        return response.read()
+
+
 def _validate_immutable_url(url: str, path: str) -> None:
     parsed = urlparse(url)
     if parsed.scheme != "https":
@@ -125,20 +212,35 @@ def _non_empty_string(value: object, path: str) -> str:
     return value
 
 
+def _optional_string(value: object, path: str) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str) or not value:
+        raise ValueError(f"{path} must be a non-empty string when defined")
+    return value
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
-    parser.add_argument("command", choices=("sync", "check"))
+    parser.add_argument("command", choices=("sync", "check", "refresh"))
     parser.add_argument("manifest", type=Path)
     args = parser.parse_args()
 
     root = Path.cwd()
-    sources = load_sources(args.manifest)
 
     try:
-        if args.command == "sync":
-            sync(sources, root)
+        if args.command == "refresh":
+            refresh(
+                args.manifest,
+                root,
+                token=os.environ.get("GITHUB_TOKEN"),
+            )
         else:
-            check(sources, root)
+            sources = load_sources(args.manifest)
+            if args.command == "sync":
+                sync(sources, root)
+            else:
+                check(sources, root)
     except ValueError as error:
         parser.error(str(error))