From 0a30acd2c400b63facdacf7c33ae62eccec16264 Mon Sep 17 00:00:00 2001 From: Michael Costello Date: Tue, 10 Nov 2026 14:06:29 -0500 Subject: [PATCH] feat: Run the catalog on the shared workflows This catalog's index rebuild and release scan pushed with the built-in token, so GitHub held every run they triggered until a maintainer approved it, and its pull request check passed without validating any manifest. The catalog now uses the shared plugin catalog workflows from datum-cloud/actions v1.27.1. The rebuild and scan push with the release bot's GitHub App, so CI runs on their pull requests without an approval, and validation checks every manifest, runs datumctl's own validator, and downloads the archives a pull request changes. Its own scripts and schema copy go. The index has no header, as before, so what datumctl reads doesn't change. Fixes #236 Fixes #37 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/catalog-health.yaml | 16 ++ .github/workflows/ci.yaml | 5 +- .github/workflows/generate-index.yaml | 54 ----- .github/workflows/rebuild-index.yaml | 18 ++ .github/workflows/scan-releases.yaml | 48 ++--- .github/workflows/validate-pr.yaml | 73 ------- README.md | 36 +++- schema/plugin-v1alpha1.json | 100 --------- scripts/ci-check | 43 ---- scripts/scan-upstream-releases.py | 294 -------------------------- scripts/sign-commit | 132 ------------ 11 files changed, 78 insertions(+), 741 deletions(-) create mode 100644 .github/workflows/catalog-health.yaml delete mode 100644 .github/workflows/generate-index.yaml create mode 100644 .github/workflows/rebuild-index.yaml delete mode 100644 .github/workflows/validate-pr.yaml delete mode 100644 schema/plugin-v1alpha1.json delete mode 100755 scripts/ci-check delete mode 100755 scripts/scan-upstream-releases.py delete mode 100755 scripts/sign-commit diff --git a/.github/workflows/catalog-health.yaml b/.github/workflows/catalog-health.yaml new file mode 100644 index 0000000..a169f39 --- /dev/null +++ b/.github/workflows/catalog-health.yaml @@ -0,0 +1,16 @@ +name: Catalog health + +on: + schedule: + - cron: "0 12 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + health: + uses: datum-cloud/actions/.github/workflows/plugin-catalog-health.yaml@v1.27.1 + permissions: + contents: read + issues: write diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7452e05..5c0aa3a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,6 +10,7 @@ permissions: jobs: ci: - uses: datum-cloud/actions/.github/workflows/ci-basic.yaml@v1.27.1 + uses: datum-cloud/actions/.github/workflows/plugin-catalog-validate.yaml@v1.27.1 with: - run: scripts/ci-check + # The branch rules require the "ci / Build and test" check. + check-name: Build and test diff --git a/.github/workflows/generate-index.yaml b/.github/workflows/generate-index.yaml deleted file mode 100644 index e7b0282..0000000 --- a/.github/workflows/generate-index.yaml +++ /dev/null @@ -1,54 +0,0 @@ -name: Regenerate index.yaml - -on: - workflow_dispatch: - push: - branches: [main] - paths: - - "plugins/**" - -permissions: - contents: write - pull-requests: write - -jobs: - generate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: Generate index.yaml from plugins/ - run: | - python3 - <<'EOF' - import os, yaml - - header = { - "apiVersion": "datumctl.datum.net/v1alpha1", - "kind": "PluginList", - "items": [], - } - - for filename in sorted(os.listdir("plugins")): - if not filename.endswith(".yaml"): - continue - with open(f"plugins/{filename}") as fh: - plugin = yaml.safe_load(fh) - header["items"].append(plugin) - - with open("index.yaml", "w") as fh: - yaml.dump(header, fh, default_flow_style=False, allow_unicode=True, sort_keys=False) - - print(f"Generated index.yaml with {len(header['items'])} plugin(s)") - EOF - - - name: Open PR with updated index.yaml - uses: peter-evans/create-pull-request@v8 - with: - commit-message: "chore: regenerate index.yaml" - branch: chore/regen-index - sign-commits: true - title: "chore: regenerate index.yaml" - body: | - The plugin registry index has been automatically regenerated following a change to one or more plugin manifests. - - The index is what `datumctl` reads to discover available plugins, their current versions, and where to download them for each platform. Keeping it in sync ensures users always see accurate information when installing or updating plugins. diff --git a/.github/workflows/rebuild-index.yaml b/.github/workflows/rebuild-index.yaml new file mode 100644 index 0000000..0c15daa --- /dev/null +++ b/.github/workflows/rebuild-index.yaml @@ -0,0 +1,18 @@ +name: Rebuild index + +on: + push: + branches: [main] + paths: + - "plugins/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + rebuild: + uses: datum-cloud/actions/.github/workflows/plugin-catalog-index.yaml@v1.27.1 + secrets: + app-id: ${{ secrets.RELEASE_BOT_APP_ID }} + app-private-key: ${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }} diff --git a/.github/workflows/scan-releases.yaml b/.github/workflows/scan-releases.yaml index e6d4098..d7af80e 100644 --- a/.github/workflows/scan-releases.yaml +++ b/.github/workflows/scan-releases.yaml @@ -1,50 +1,28 @@ -# Notices when a plugin's upstream repo has published a newer release than this -# catalog offers, and opens the PR to catch up. -# -# A plugin's own repo is supposed to push its release here. When that push -# breaks the catalog goes quietly stale and people install an old client. This -# runs regardless of why the push did not happen. -# -# The work is in scripts/scan-upstream-releases.py so it can be read, reviewed -# and run by hand. -name: Scan upstream releases +name: Scan releases on: schedule: - # Daily, early UTC. Nothing here is urgent enough to run more often. - cron: "17 6 * * *" workflow_dispatch: inputs: - dry_run: - description: "Report what is behind without opening PRs" + dry-run: + description: Report which plugins are behind without opening pull requests type: boolean default: false plugin: - description: "Only this plugin (leave empty for all)" + description: Scan only this plugin, by name type: string - required: false + default: "" permissions: - contents: write - pull-requests: write + contents: read jobs: scan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - uses: actions/setup-python@v7 - with: - python-version: "3.x" - - - name: Install PyYAML - run: pip install --quiet pyyaml - - - name: Scan for newer releases - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - scripts/scan-upstream-releases.py \ - ${{ inputs.dry_run && '--dry-run' || '' }} \ - ${{ inputs.plugin && format('--plugin {0}', inputs.plugin) || '' }} + uses: datum-cloud/actions/.github/workflows/plugin-catalog-scan.yaml@v1.27.1 + with: + dry-run: ${{ inputs.dry-run || false }} + plugin: ${{ inputs.plugin || '' }} + secrets: + app-id: ${{ secrets.RELEASE_BOT_APP_ID }} + app-private-key: ${{ secrets.RELEASE_BOT_APP_PRIVATE_KEY }} diff --git a/.github/workflows/validate-pr.yaml b/.github/workflows/validate-pr.yaml deleted file mode 100644 index f819a4a..0000000 --- a/.github/workflows/validate-pr.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: Validate plugin manifests - -on: - pull_request: - branches: [main] - paths: - - "plugins/**" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' || (startsWith(github.ref, 'refs/heads/') && github.ref != format('refs/heads/{0}', github.event.repository.default_branch)) }} - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: Install ajv-cli - run: npm install -g ajv-cli ajv-formats - - - name: Find changed plugin manifests - id: changed - run: | - files=$(git diff --name-only origin/main...HEAD -- plugins/ | tr '\n' ' ') - echo "files=$files" >> "$GITHUB_OUTPUT" - - - name: Convert YAML manifests to JSON and validate schema - run: | - for f in ${{ steps.changed.outputs.files }}; do - echo "Validating schema: $f" - python3 -c " - import sys, json, yaml - with open('$f') as fh: - data = yaml.safe_load(fh) - print(json.dumps(data)) - " > /tmp/manifest.json - ajv validate -s schema/plugin-v1alpha1.json -d /tmp/manifest.json --spec=draft7 - done - - - name: Check URIs resolve and verify SHA256 - run: | - for f in ${{ steps.changed.outputs.files }}; do - echo "Checking URIs in: $f" - python3 - <<'EOF' - import sys, yaml, urllib.request, hashlib - - with open("$f") as fh: - manifest = yaml.safe_load(fh) - - for platform in manifest["spec"]["platforms"]: - uri = platform["uri"] - expected_sha = platform["sha256"] - print(f" Checking {uri}") - - req = urllib.request.Request(uri, method="HEAD") - with urllib.request.urlopen(req) as resp: - if resp.status != 200: - print(f" ERROR: {uri} returned {resp.status}", file=sys.stderr) - sys.exit(1) - - print(f" Downloading and verifying SHA256...") - with urllib.request.urlopen(uri) as resp: - data = resp.read() - actual_sha = hashlib.sha256(data).hexdigest() - if actual_sha != expected_sha: - print(f" ERROR: SHA256 mismatch for {uri}", file=sys.stderr) - print(f" expected: {expected_sha}", file=sys.stderr) - print(f" actual: {actual_sha}", file=sys.stderr) - sys.exit(1) - print(f" OK: {actual_sha}") - EOF - done diff --git a/README.md b/README.md index 3a19de8..d66202a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# datumctl-plugins +# Datum Cloud plugins -The official plugin index for [datumctl](https://github.com/datum-cloud/datumctl), the CLI for [Datum Cloud](https://datum.net). +The official plugin catalog for [datumctl](https://github.com/datum-cloud/datumctl), the CLI for [Datum Cloud](https://datum.net). `datumctl` reads this catalog by default, so every plugin listed here installs by name with no setup. ## Available plugins @@ -16,19 +16,33 @@ The official plugin index for [datumctl](https://github.com/datum-cloud/datumctl | [search](plugins/search.yaml) | Search for resources across the platform by kind, name, and project scope | | [telemetry](plugins/telemetry.yaml) | Query logs for your Datum Cloud projects | -## Installing a plugin +## Using this catalog ```sh +# Search and install +datumctl plugin search compute datumctl plugin install compute -datumctl plugin install ipam + +# Or browse interactively +datumctl plugin browse +``` + +Once installed, a plugin runs as a `datumctl` subcommand: + +```sh +datumctl compute --help ``` +Plugins for the Milo platform live in the [Milo catalog](https://github.com/milo-os/cli-plugins), which you add with `datumctl plugin index add milo-os milo-os/cli-plugins`. + ## Submitting a plugin -1. Add the `datumctl-plugin` topic to your GitHub repository. -2. Open a PR adding `plugins/.yaml` following the [schema](schema/plugin-v1alpha1.json). -3. CI will validate your manifest: schema, URI resolution, and SHA256 verification. -4. Once merged, `index.yaml` is regenerated automatically. +1. Add the `datumctl-plugin` topic to your plugin's GitHub repository. +2. Open a pull request adding `plugins/.yaml`, following the [schema](https://github.com/datum-cloud/actions/blob/v1.27.1/scripts/plugin-catalog/schema/plugin-v1alpha1.json). That's the only file you edit. +3. CI validates every manifest against the schema and `datumctl plugin index validate`, and checks that each changed download URL resolves and matches its SHA256. A weekly health check re-verifies the whole catalog. +4. On merge, [`index.yaml`](index.yaml), the file `datumctl` actually reads, is rebuilt from `plugins/*.yaml` through a pull request. You never edit it by hand. + +New releases reach this catalog through the [`update-plugin-index`](https://github.com/datum-cloud/actions/blob/main/docs/update-plugin-index/README.md) action in your release workflow, and a daily scan opens an update for any release that action missed. ## Plugin manifest format @@ -50,6 +64,8 @@ spec: sha256: ``` +### Binary naming + The binary inside each archive must be named `datumctl-` or `milo-` (add `.exe` on Windows). datumctl recognizes both prefixes, so a plugin built for the milo-os platform installs from this catalog without a second, datumctl-branded build. @@ -59,3 +75,7 @@ To explicitly select the binary to install, set `files` for each platform, as in files: - from: milo-ipam # milo-ipam.exe on Windows ``` + +## How this catalog runs + +Validation, the index rebuild, the daily release scan, and the weekly health check all come from the shared [plugin catalog workflows](https://github.com/datum-cloud/actions/blob/main/docs/plugin-catalog/README.md) in `datum-cloud/actions`. diff --git a/schema/plugin-v1alpha1.json b/schema/plugin-v1alpha1.json deleted file mode 100644 index 753efa0..0000000 --- a/schema/plugin-v1alpha1.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Plugin", - "description": "A datumctl plugin manifest (apiVersion: datumctl.datum.net/v1alpha1, kind: Plugin)", - "type": "object", - "required": ["apiVersion", "kind", "metadata", "spec"], - "properties": { - "apiVersion": { - "type": "string", - "const": "datumctl.datum.net/v1alpha1" - }, - "kind": { - "type": "string", - "const": "Plugin" - }, - "metadata": { - "type": "object", - "required": ["name"], - "properties": { - "name": { - "type": "string", - "pattern": "^[a-z][a-z0-9-]*$", - "description": "Short plugin name, e.g. 'compute'. Must be lowercase alphanumeric with hyphens." - } - } - }, - "spec": { - "type": "object", - "required": ["shortDescription", "version", "platforms"], - "properties": { - "shortDescription": { - "type": "string", - "maxLength": 120, - "description": "One-line summary shown in datumctl plugin search." - }, - "description": { - "type": "string", - "description": "Optional longer help text." - }, - "homepage": { - "type": "string", - "format": "uri" - }, - "version": { - "type": "string", - "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]", - "description": "Semantic version string, e.g. v1.2.0" - }, - "platforms": { - "type": "array", - "minItems": 1, - "items": { - "type": "object", - "required": ["uri", "sha256"], - "properties": { - "selector": { - "type": "object", - "properties": { - "matchLabels": { - "type": "object", - "properties": { - "os": { - "type": "string", - "enum": ["linux", "darwin", "windows"] - }, - "arch": { - "type": "string", - "enum": ["amd64", "arm64", "386"] - } - } - } - } - }, - "uri": { - "type": "string", - "format": "uri" - }, - "sha256": { - "type": "string", - "pattern": "^[a-f0-9]{64}$", - "description": "Lowercase hex SHA256 of the archive at uri." - }, - "files": { - "type": "array", - "items": { - "type": "object", - "required": ["from"], - "properties": { - "from": { "type": "string" }, - "to": { "type": "string" } - } - } - } - } - } - } - } - } - } -} diff --git a/scripts/ci-check b/scripts/ci-check deleted file mode 100755 index a328c77..0000000 --- a/scripts/ci-check +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -cd "$(dirname "$0")/.." - -venv=$(mktemp -d) -trap 'rm -rf "$venv"' EXIT -python3 -m venv "$venv" -"$venv/bin/pip" install --quiet 'jsonschema==4.26.0' 'PyYAML==6.0.3' - -"$venv/bin/python" -c ' -import glob, json, sys -import yaml -from jsonschema import Draft7Validator - -with open("schema/plugin-v1alpha1.json") as fh: - validator = Draft7Validator(json.load(fh)) - -paths = sorted(glob.glob("plugins/*.yaml")) -if not paths: - sys.exit("ci-check: found zero plugin manifests under plugins/") - -failed = False -for path in paths: - with open(path) as fh: - manifest = yaml.safe_load(fh) - errors = sorted(validator.iter_errors(manifest), key=lambda e: list(e.path)) - for e in errors: - loc = "/".join(str(p) for p in e.path) or "(root)" - print(f"{path}: {loc}: {e.message}", file=sys.stderr) - failed = failed or bool(errors) - print(path + (": FAIL" if errors else ": ok")) - -with open("index.yaml") as fh: - yaml.safe_load(fh) -sys.exit(1 if failed else 0) -' - -"$venv/bin/python" -c ' -import sys -with open(sys.argv[1]) as fh: - compile(fh.read(), sys.argv[1], "exec") -' scripts/scan-upstream-releases.py diff --git a/scripts/scan-upstream-releases.py b/scripts/scan-upstream-releases.py deleted file mode 100755 index 83d5efe..0000000 --- a/scripts/scan-upstream-releases.py +++ /dev/null @@ -1,294 +0,0 @@ -#!/usr/bin/env python3 -"""Open a PR for any plugin whose upstream repo has published a newer release. - -A plugin's own repo normally pushes its release into this catalog. When that -push breaks the catalog goes stale and people install an old client, with -nothing to say so. This is the backstop, and it does not care why the push -did not happen. - - scripts/scan-upstream-releases.py --dry-run # report, change nothing - scripts/scan-upstream-releases.py --plugin assistant # just the one -""" - -import argparse -import hashlib -import json -import os -import re -import subprocess -import sys -import urllib.request - -import yaml - -PLUGINS_DIR = "plugins" -RELEASE_URI = re.compile(r"https://github\.com/([^/]+/[^/]+)/releases/download/") - - -def run(*args): - return subprocess.run(args, capture_output=True, text=True) - - -def latest_release(repo): - """Newest published release for a repo. Prereleases and drafts are not it.""" - result = run("gh", "api", f"repos/{repo}/releases/latest", "--jq", ".tag_name") - return result.stdout.strip() if result.returncode == 0 else None - - -def published_checksums(repo, tag): - """asset name -> sha256, from the release's checksums.txt if it has one.""" - out = run("gh", "release", "download", tag, "--repo", repo, - "--pattern", "checksums.txt", "--dir", "/tmp/upstream-sums", "--clobber") - if out.returncode != 0: - return {} - sums = {} - with open("/tmp/upstream-sums/checksums.txt") as handle: - for line in handle: - digest, _, name = line.strip().partition(" ") - if name: - sums[name] = digest - return sums - - -def sha256_of(url): - """For releases that publish no checksums.txt.""" - with urllib.request.urlopen(url) as response: - digest = hashlib.sha256() - for chunk in iter(lambda: response.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() - - -def repo_for(manifest): - """The repo a plugin's archives come from, or None if not GitHub releases.""" - match = RELEASE_URI.match(manifest["spec"]["platforms"][0]["uri"]) - return match.group(1) if match else None - - -def retarget(text, old, new, sums, on_missing): - """Point a manifest at `new`, giving each platform its own archive's digest. - - Pairs every uri with the sha256 line that follows it. Getting that pairing - wrong would hand one platform another's digest, which installs cleanly for - nobody. - """ - text = text.replace(f"version: {old}", f"version: {new}") - text = text.replace(f"/releases/download/{old}/", f"/releases/download/{new}/") - - lines, pending = [], None - for line in text.split("\n"): - uri = re.match(r"^\s*uri: (\S+)$", line) - if uri: - pending = uri.group(1) - digest = re.match(r"^(\s*)sha256: \w+$", line) - if digest and pending: - asset = pending.rsplit("/", 1)[-1] - value = sums.get(asset) or on_missing(pending) - line = f"{digest.group(1)}sha256: {value}" - pending = None - lines.append(line) - return "\n".join(lines) - - -def pr_body(name, repo, current, newest): - return ( - f"`{name}` is on {current} here; {repo} has published **{newest}**.\n\n" - "Opened or updated by the daily upstream scan. A plugin's own repo normally pushes " - "its release to this catalog; this runs regardless of whether that " - "happened, so the catalog cannot drift unnoticed.\n\n" - "Checksums come from the release's `checksums.txt` where it has one, and " - "are computed from the archives where it does not. PR validation " - "downloads every URI and verifies them again." - ) - - -def catalog_repo(): - """owner/name of this catalog: Actions sets GITHUB_REPOSITORY; by hand, gh knows.""" - return os.environ.get("GITHUB_REPOSITORY") or run( - "gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner").stdout.strip() - - -def open_pulls(catalog): - """Open PRs into main from the catalog's own branches: number, branch, head commit.""" - jq = (f'.[] | select((.head.repo.full_name // "" | ascii_downcase) == "{catalog.lower()}")' - " | {number, branch: .head.ref, sha: .head.sha}") - out = run("gh", "api", "--paginate", f"repos/{catalog}/pulls?state=open&base=main&per_page=100", "--jq", jq) - if out.returncode != 0: - # Guessing "no open PRs" here would open duplicates, the thing this avoids. - sys.exit(f"could not list the open PRs: {out.stderr.strip()}") - return [json.loads(line) for line in out.stdout.splitlines()] - - -def pr_files(catalog, number): - out = run("gh", "api", "--paginate", f"repos/{catalog}/pulls/{number}/files?per_page=100", "--jq", ".[].filename") - return out.stdout.split() - - -def candidates(pulls, stable): - """PRs on the stable branch first, then on its versioned branches, oldest first.""" - mine = [pr for pr in pulls if pr["branch"] == stable or pr["branch"].startswith(stable + "-")] - return sorted(mine, key=lambda pr: (pr["branch"] != stable, pr["number"])) - - -def plugin_pr(catalog, pulls, name, path): - """The open PR that carries this plugin's updates, chosen as the release action chooses it. - - datum-cloud/actions update-plugin-index (v1.22.0, resolve-pr-branch.sh) reuses the PR on - update-plugin/, else the oldest on update-plugin/-*, when the plugin's manifest - is its only file. Choosing the same PR keeps one open update per plugin, whoever opened it. - """ - for pr in candidates(pulls, f"update-plugin/{name}"): - if pr_files(catalog, pr["number"]) == [path]: - return pr - return None - - -def proposed_version(catalog, pr, path): - """The plugin's version at the PR's head.""" - out = run("gh", "api", "-H", "Accept: application/vnd.github.raw", - f"repos/{catalog}/contents/{path}?ref={pr['sha']}") - return yaml.safe_load(out.stdout)["spec"]["version"] if out.returncode == 0 else None - - -def stable_taken(pulls, name): - return any(pr["branch"] == f"update-plugin/{name}" for pr in pulls) - - -def choose_pr(catalog, pulls, name, path, newest): - """The PR to update (None: open one), and why to leave the plugin alone, if so.""" - pr = plugin_pr(catalog, pulls, name, path) - if pr and proposed_version(catalog, pr, path) == newest: - return pr, f"#{pr['number']} already proposes {newest}" - if not pr and stable_taken(pulls, name): - # The release action refuses this case too, rather than overwrite another PR. - return None, f"update-plugin/{name} carries a PR that changes more than {path}" - return pr, None - - -def plan(pr, why, name): - """What the scan does about a plugin that is behind, in a few words.""" - if why: - return why - return f"update #{pr['number']}" if pr else f"open a PR on update-plugin/{name}" - - -def publish(path, name, newest, pr, body): - """Commit the manifest to the plugin's PR branch, then open or retitle the PR. - - The commit goes through the API so GitHub signs it. A reused branch is reset to main first, - as the release action rewrites it. Returns why it failed, or None. - """ - branch = pr["branch"] if pr else f"update-plugin/{name}" - title = f"Update the {name} plugin to {newest}" - base = run("git", "rev-parse", "main").stdout.strip() - env = {k: v for k, v in os.environ.items() if k != "EXPECTED_HEAD"} - env.update(BASE_SHA=base, GITHUB_REPOSITORY=catalog_repo()) - commit = subprocess.run(["scripts/sign-commit", branch, f"chore: {title}", path], - capture_output=True, text=True, env=env) - run("git", "checkout", "--", path) - if commit.returncode != 0: - return f"signed commit failed: {commit.stderr.strip()}" - if pr: - run("gh", "pr", "edit", str(pr["number"]), "--title", title, "--body", body) - else: - run("gh", "pr", "create", "--base", "main", "--head", branch, "--title", title, "--body", body) - return None - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--dry-run", action="store_true", - help="report what would change without touching git") - parser.add_argument("--plugin", help="only this plugin, by name") - args = parser.parse_args() - - opened, skipped, behind = [], [], [] - matched = False - catalog = catalog_repo() - pulls = open_pulls(catalog) - - for filename in sorted(os.listdir(PLUGINS_DIR)): - if not filename.endswith(".yaml"): - continue - path = os.path.join(PLUGINS_DIR, filename) - with open(path) as handle: - manifest = yaml.safe_load(handle) - name = manifest["metadata"]["name"] - if args.plugin and name != args.plugin: - continue - matched = True - - current = manifest["spec"]["version"] - repo = repo_for(manifest) - if not repo: - skipped.append(f"{name}: archives are not on GitHub releases") - continue - - newest = latest_release(repo) - if not newest: - skipped.append(f"{name}: {repo} has no published release") - continue - if newest == current: - continue - - pr, why = choose_pr(catalog, pulls, name, path, newest) - behind.append(f"{name}: {current} -> {newest} (from {repo}): {plan(pr, why, name)}") - if args.dry_run: - continue - if why: - skipped.append(f"{name}: {why}") - continue - - failures = [] - - def on_missing(url, failures=failures): - try: - return sha256_of(url) - except Exception as exc: # noqa: BLE001 - reported, not raised - failures.append(f"{url.rsplit('/', 1)[-1]}: {exc}") - return "unknown" - - rewritten = retarget(open(path).read(), current, newest, - published_checksums(repo, newest), on_missing) - if failures: - skipped.append(f"{name}: could not hash {'; '.join(failures)}") - continue - - with open(path, "w") as handle: - handle.write(rewritten) - - failure = publish(path, name, newest, pr, pr_body(name, repo, current, newest)) - if failure: - skipped.append(f"{name}: {failure}") - continue - opened.append(f"{name}: {current} -> {newest} ({plan(pr, None, name)})") - - # A typo in --plugin must not read as an all-clear. - if args.plugin and not matched: - print(f"no plugin named {args.plugin!r} in {PLUGINS_DIR}/", file=sys.stderr) - return 1 - - report(args, opened, skipped, behind) - return 0 - - -def report(args, opened, skipped, behind): - lines = ["## Upstream release scan", ""] - if args.dry_run: - lines += [f"- behind: {item}" for item in behind] or ["- everything is up to date"] - lines += ["", "_Dry run: nothing was changed._"] - else: - lines += [f"- proposed {item}" for item in opened] or ["- nothing to update"] - if skipped: - lines += ["", "### Skipped", ""] + [f"- {item}" for item in skipped] - - text = "\n".join(lines) - print(text) - summary = os.environ.get("GITHUB_STEP_SUMMARY") - if summary: - with open(summary, "a") as handle: - handle.write(text + "\n") - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/sign-commit b/scripts/sign-commit deleted file mode 100755 index 2082bbe..0000000 --- a/scripts/sign-commit +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# Commit files to a branch through the GitHub API so GitHub signs the commit. -# Usage: sign-commit [-z] BRANCH MESSAGE [PATH...] -# A PATH that is a file is committed with its current contents; one that does -# not exist at all is deleted from the branch. Directories and symlinks are -# refused. With -z, further PATHs are read from stdin, NUL-separated, so -# `git diff -z --name-only` output passes through without quoting or splitting. -# Exits 1 when there is nothing to commit, and 3 when the remote branch is not -# at the expected head, so a caller can refetch and try again. -# Set EXPECTED_HEAD to the commit the job checked out: the commit fails unless -# the remote branch is still there. Set BASE_SHA instead to create BRANCH at -# that commit (or reset it there) first. One of the two is required. -# Needs GH_TOKEN and GITHUB_REPOSITORY. -set -euo pipefail - -from_stdin=0 -if [ "${1:-}" = "-z" ]; then - from_stdin=1 - shift -fi -if [ $# -lt 2 ]; then - echo "Usage: sign-commit [-z] BRANCH MESSAGE [PATH...]" >&2 - exit 2 -fi - -branch=$1 -message=$2 -shift 2 - -repo=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required} -if [ -z "${BASE_SHA:-}" ] && [ -z "${EXPECTED_HEAD:-}" ]; then - echo "sign-commit: set EXPECTED_HEAD (or BASE_SHA to create the branch)." >&2 - exit 2 -fi -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT - -: > "$work/additions" -: > "$work/deletions" -n=0 -adds=0 -dels=0 - -add_path() { - local path=$1 - if [ -L "$path" ]; then - echo "sign-commit: ${path} is a symlink; the API cannot commit one." >&2 - exit 1 - elif [ -d "$path" ]; then - echo "sign-commit: ${path} is a directory; pass the files in it." >&2 - exit 1 - elif [ -f "$path" ]; then - n=$((n + 1)) - adds=$((adds + 1)) - base64 < "$path" | tr -d '\n' > "$work/$n.b64" - jq -nc --arg path "$path" --rawfile contents "$work/$n.b64" '{path: $path, contents: $contents}' >> "$work/additions" - elif [ ! -e "$path" ]; then - dels=$((dels + 1)) - jq -nc --arg path "$path" '{path: $path}' >> "$work/deletions" - else - echo "sign-commit: ${path} is not a regular file." >&2 - exit 1 - fi -} - -for path in "$@"; do - add_path "$path" -done -if [ "$from_stdin" = 1 ]; then - while IFS= read -r -d '' path; do - add_path "$path" - done -fi - -if [ $((adds + dels)) -eq 0 ]; then - echo "sign-commit: nothing to commit." >&2 - exit 1 -fi - -remote_head() { - gh api "repos/${repo}/git/ref/heads/${branch}" --jq .object.sha -} - -if [ -n "${BASE_SHA:-}" ]; then - if ! gh api -X POST "repos/${repo}/git/refs" -f "ref=refs/heads/${branch}" -f "sha=${BASE_SHA}" >/dev/null 2>&1; then - gh api -X PATCH "repos/${repo}/git/refs/heads/${branch}" -f "sha=${BASE_SHA}" -F force=true >/dev/null - fi - expected=$BASE_SHA -else - expected=$EXPECTED_HEAD - head=$(remote_head) - if [ "$head" != "$expected" ]; then - echo "sign-commit: ${branch} is at ${head}, but this job checked out ${expected}." >&2 - exit 3 - fi -fi - -headline=${message%%$'\n'*} -body="" -if [ "$headline" != "$message" ]; then - body=${message#*$'\n'} - body=${body#$'\n'} -fi - -query='mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { url } } }' - -jq -n --arg query "$query" --arg repo "$repo" --arg branch "$branch" --arg head "$expected" \ - --arg headline "$headline" --arg body "$body" \ - --slurpfile additions "$work/additions" --slurpfile deletions "$work/deletions" '{ - query: $query, - variables: {input: { - branch: {repositoryNameWithOwner: $repo, branchName: $branch}, - expectedHeadOid: $head, - message: ({headline: $headline} + (if $body == "" then {} else {body: $body} end)), - fileChanges: {additions: $additions, deletions: $deletions} - }} - }' > "$work/request.json" - -for attempt in 1 2 3 4 5; do - if gh api graphql --input "$work/request.json" --jq .data.createCommitOnBranch.commit.url; then - exit 0 - fi - head=$(remote_head) - if [ "$head" != "$expected" ]; then - echo "sign-commit: ${branch} moved to ${head} while committing; not retrying." >&2 - exit 3 - fi - echo "Attempt ${attempt} failed, retrying." >&2 - sleep $((attempt * 2)) -done -echo "Could not commit to ${branch}." >&2 -exit 1