From 52ead71d748d67e593f76a7986c8824acd5a2ef7 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 22 Jul 2026 11:20:30 -0400 Subject: [PATCH 01/13] Replace workflow/shell automation with a tested pluginctl CLI The registry automation had grown to roughly 5,000 lines of inline YAML and Bash across the plugin workflows, with the app-token block copy-pasted 6 times, the fork-checkout dance 3 times, and the CodeQL SARIF jq program pasted 3 times. This moves the logic into a testable Python package and leaves the workflows as thin wiring. - Add .github/pluginctl: a stdlib-only Python package (core, validate, publish, integrations) exposing one CLI subcommand per task, with a 113-test pytest suite covering the pure logic and golden PR-comment/manifest/README output. - Add two composite actions (gh-app-token, trusted-checkout) and two reusable workflows (_codeql-scan, _clamav-scan) to remove the biggest duplication. - Thin the five core workflows to composite-action + `python -m pluginctl` steps; validate-plugin.yml drops from 1,370 to 442 lines. - Add an opt-in, best-effort signed webhook emitter (HMAC-SHA256) plus docs/webhooks.md, so a downstream consumer can react to registry events. - Delete the V1 validate/publish shell scripts; keep scripts/keys. - Remove the disabled standalone codeql workflow. - Fix a latent bug: deprecated plugins' version count now derives from the manifest's versions[] instead of the removed zips/ path (which rendered 0). User-facing output is kept byte-identical to the old shell, verified by the golden tests. Final fork end-to-end validation is still pending. --- .github/actions/gh-app-token/action.yml | 85 ++ .github/actions/trusted-checkout/action.yml | 64 + .github/pluginctl/README.md | 79 + .github/pluginctl/fixtures/sample.sarif | 43 + .github/pluginctl/pyproject.toml | 24 + .github/pluginctl/src/pluginctl/__init__.py | 3 + .github/pluginctl/src/pluginctl/__main__.py | 6 + .github/pluginctl/src/pluginctl/cli.py | 270 ++++ .../pluginctl/src/pluginctl/core/__init__.py | 1 + .../pluginctl/src/pluginctl/core/actions.py | 66 + .github/pluginctl/src/pluginctl/core/gh.py | 183 +++ .github/pluginctl/src/pluginctl/core/git.py | 80 ++ .../pluginctl/src/pluginctl/core/jsonio.py | 38 + .../pluginctl/src/pluginctl/core/models.py | 97 ++ .../pluginctl/src/pluginctl/core/version.py | 89 ++ .../src/pluginctl/integrations/__init__.py | 1 + .../src/pluginctl/integrations/automerge.py | 111 ++ .../pluginctl/integrations/external_readme.py | 157 ++ .../src/pluginctl/integrations/webhooks.py | 118 ++ .../src/pluginctl/publish/__init__.py | 1 + .../src/pluginctl/publish/cleanup.py | 47 + .../src/pluginctl/publish/manifest.py | 399 ++++++ .../src/pluginctl/publish/readmes.py | 440 ++++++ .../pluginctl/src/pluginctl/publish/run.py | 215 +++ .../pluginctl/src/pluginctl/publish/yank.py | 264 ++++ .../pluginctl/src/pluginctl/publish/zips.py | 145 ++ .../src/pluginctl/validate/__init__.py | 1 + .../src/pluginctl/validate/clamav.py | 78 + .../src/pluginctl/validate/detect.py | 190 +++ .../pluginctl/src/pluginctl/validate/gate.py | 43 + .../src/pluginctl/validate/labels.py | 35 + .../pluginctl/src/pluginctl/validate/langs.py | 79 + .../src/pluginctl/validate/plugin.py | 394 +++++ .../src/pluginctl/validate/report.py | 425 ++++++ .../pluginctl/src/pluginctl/validate/sarif.py | 259 ++++ .../pluginctl/src/pluginctl/validate/title.py | 69 + .github/pluginctl/tests/conftest.py | 6 + .github/pluginctl/tests/test_clamav.py | 28 + .github/pluginctl/tests/test_detect.py | 48 + .github/pluginctl/tests/test_integrations.py | 60 + .github/pluginctl/tests/test_jsonio.py | 46 + .github/pluginctl/tests/test_labels_gate.py | 55 + .github/pluginctl/tests/test_langs.py | 45 + .github/pluginctl/tests/test_manifest.py | 103 ++ .github/pluginctl/tests/test_readmes.py | 55 + .github/pluginctl/tests/test_report.py | 86 ++ .github/pluginctl/tests/test_sarif.py | 73 + .github/pluginctl/tests/test_title.py | 48 + .github/pluginctl/tests/test_validate.py | 109 ++ .github/pluginctl/tests/test_version.py | 36 + .github/pluginctl/tests/test_webhooks.py | 59 + .github/scripts/publish/build-zips.sh | 149 -- .github/scripts/publish/cleanup.sh | 58 - .github/scripts/publish/generate-manifest.sh | 355 ----- .github/scripts/publish/plugin-readmes.sh | 172 --- .github/scripts/publish/releases-readme.sh | 253 ---- .github/scripts/publish/run.sh | 203 --- .github/scripts/publish/yank-version.sh | 283 ---- .github/scripts/validate/detect-changes.sh | 224 --- .github/scripts/validate/report.sh | 405 ------ .github/scripts/validate/validate.sh | 425 ------ .github/workflows/_clamav-scan.yml | 182 +++ .github/workflows/_codeql-scan.yml | 158 ++ .github/workflows/auto-merge-updates.yml | 161 +-- .github/workflows/codeql.yml | 36 - .github/workflows/publish-plugins.yml | 58 +- .github/workflows/update-external-readme.yml | 183 +-- .github/workflows/validate-plugin.yml | 1275 ++--------------- .github/workflows/yank-plugin-version.yml | 39 +- docs/webhooks.md | 82 ++ 70 files changed, 6066 insertions(+), 4091 deletions(-) create mode 100644 .github/actions/gh-app-token/action.yml create mode 100644 .github/actions/trusted-checkout/action.yml create mode 100644 .github/pluginctl/README.md create mode 100644 .github/pluginctl/fixtures/sample.sarif create mode 100644 .github/pluginctl/pyproject.toml create mode 100644 .github/pluginctl/src/pluginctl/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/__main__.py create mode 100644 .github/pluginctl/src/pluginctl/cli.py create mode 100644 .github/pluginctl/src/pluginctl/core/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/core/actions.py create mode 100644 .github/pluginctl/src/pluginctl/core/gh.py create mode 100644 .github/pluginctl/src/pluginctl/core/git.py create mode 100644 .github/pluginctl/src/pluginctl/core/jsonio.py create mode 100644 .github/pluginctl/src/pluginctl/core/models.py create mode 100644 .github/pluginctl/src/pluginctl/core/version.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/automerge.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/external_readme.py create mode 100644 .github/pluginctl/src/pluginctl/integrations/webhooks.py create mode 100644 .github/pluginctl/src/pluginctl/publish/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/publish/cleanup.py create mode 100644 .github/pluginctl/src/pluginctl/publish/manifest.py create mode 100644 .github/pluginctl/src/pluginctl/publish/readmes.py create mode 100644 .github/pluginctl/src/pluginctl/publish/run.py create mode 100644 .github/pluginctl/src/pluginctl/publish/yank.py create mode 100644 .github/pluginctl/src/pluginctl/publish/zips.py create mode 100644 .github/pluginctl/src/pluginctl/validate/__init__.py create mode 100644 .github/pluginctl/src/pluginctl/validate/clamav.py create mode 100644 .github/pluginctl/src/pluginctl/validate/detect.py create mode 100644 .github/pluginctl/src/pluginctl/validate/gate.py create mode 100644 .github/pluginctl/src/pluginctl/validate/labels.py create mode 100644 .github/pluginctl/src/pluginctl/validate/langs.py create mode 100644 .github/pluginctl/src/pluginctl/validate/plugin.py create mode 100644 .github/pluginctl/src/pluginctl/validate/report.py create mode 100644 .github/pluginctl/src/pluginctl/validate/sarif.py create mode 100644 .github/pluginctl/src/pluginctl/validate/title.py create mode 100644 .github/pluginctl/tests/conftest.py create mode 100644 .github/pluginctl/tests/test_clamav.py create mode 100644 .github/pluginctl/tests/test_detect.py create mode 100644 .github/pluginctl/tests/test_integrations.py create mode 100644 .github/pluginctl/tests/test_jsonio.py create mode 100644 .github/pluginctl/tests/test_labels_gate.py create mode 100644 .github/pluginctl/tests/test_langs.py create mode 100644 .github/pluginctl/tests/test_manifest.py create mode 100644 .github/pluginctl/tests/test_readmes.py create mode 100644 .github/pluginctl/tests/test_report.py create mode 100644 .github/pluginctl/tests/test_sarif.py create mode 100644 .github/pluginctl/tests/test_title.py create mode 100644 .github/pluginctl/tests/test_validate.py create mode 100644 .github/pluginctl/tests/test_version.py create mode 100644 .github/pluginctl/tests/test_webhooks.py delete mode 100644 .github/scripts/publish/build-zips.sh delete mode 100644 .github/scripts/publish/cleanup.sh delete mode 100644 .github/scripts/publish/generate-manifest.sh delete mode 100644 .github/scripts/publish/plugin-readmes.sh delete mode 100644 .github/scripts/publish/releases-readme.sh delete mode 100644 .github/scripts/publish/run.sh delete mode 100644 .github/scripts/publish/yank-version.sh delete mode 100755 .github/scripts/validate/detect-changes.sh delete mode 100755 .github/scripts/validate/report.sh delete mode 100755 .github/scripts/validate/validate.sh create mode 100644 .github/workflows/_clamav-scan.yml create mode 100644 .github/workflows/_codeql-scan.yml delete mode 100644 .github/workflows/codeql.yml create mode 100644 docs/webhooks.md diff --git a/.github/actions/gh-app-token/action.yml b/.github/actions/gh-app-token/action.yml new file mode 100644 index 00000000..4bf5bf42 --- /dev/null +++ b/.github/actions/gh-app-token/action.yml @@ -0,0 +1,85 @@ +name: GitHub App token +description: >- + Mint a GitHub App installation token when GH_APP_ID + GH_APP_PRIVATE_KEY are + configured, otherwise fall back to the workflow GITHUB_TOKEN + (github-actions[bot]) with the same step-summary warning. Replaces the token + block that was copy-pasted across the plugin workflows. + +inputs: + app-id: + description: GitHub App client/app id (pass vars.GH_APP_ID) + required: false + default: "" + private-key: + description: >- + GitHub App private key (pass secrets.GH_APP_PRIVATE_KEY). Composite actions + do not inherit secrets, so it must be provided as an input. + required: false + default: "" + owner: + description: Optional owner to scope the installation token to. + required: false + default: "" + +outputs: + token: + description: The App installation token, or the fallback github.token. + value: ${{ steps.result.outputs.token }} + app-slug: + description: The App slug (empty when falling back). + value: ${{ steps.app-token.outputs.app-slug }} + use_app: + description: "'true' when an App token was minted, 'false' on fallback." + value: ${{ steps.config.outputs.use_app }} + bot-login: + description: The bot login to attribute actions to. + value: ${{ steps.result.outputs.bot_login }} + +runs: + using: composite + steps: + - name: Resolve configuration + id: config + shell: bash + env: + GH_APP_ID: ${{ inputs.app-id }} + GH_APP_PRIVATE_KEY: ${{ inputs.private-key }} + run: | + if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then + echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" + echo "use_app=true" >> "$GITHUB_OUTPUT" + else + echo "use_app=false" >> "$GITHUB_OUTPUT" + fi + + - name: Generate GitHub App token + if: steps.config.outputs.use_app == 'true' + id: app-token + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ steps.config.outputs.app_id }} + private-key: ${{ inputs.private-key }} + owner: ${{ inputs.owner }} + + - name: Log fallback to actions token + if: steps.config.outputs.use_app != 'true' + shell: bash + run: | + printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" + + - name: Expose token + bot login + id: result + shell: bash + env: + USE_APP: ${{ steps.config.outputs.use_app }} + APP_TOKEN: ${{ steps.app-token.outputs.token }} + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + FALLBACK_TOKEN: ${{ github.token }} + run: | + if [[ "$USE_APP" == "true" ]]; then + echo "token=$APP_TOKEN" >> "$GITHUB_OUTPUT" + echo "bot_login=${APP_SLUG}[bot]" >> "$GITHUB_OUTPUT" + else + echo "token=$FALLBACK_TOKEN" >> "$GITHUB_OUTPUT" + echo "bot_login=github-actions[bot]" >> "$GITHUB_OUTPUT" + fi diff --git a/.github/actions/trusted-checkout/action.yml b/.github/actions/trusted-checkout/action.yml new file mode 100644 index 00000000..8feda69d --- /dev/null +++ b/.github/actions/trusted-checkout/action.yml @@ -0,0 +1,64 @@ +name: Trusted checkout +description: >- + The pull_request_target checkout "dance": check out the base branch's trusted + code, sparse-checkout the untrusted fork's plugins/ as data only, then restore + the trusted code and re-fetch the base refs. Fork content is never executed; + pluginctl always comes from the base branch. Replaces the sequence duplicated + across detect-changes / validate-plugin. + +inputs: + base-ref: + description: PR base branch (github.event.pull_request.base.ref) + required: true + head-repo: + description: Fork repo full name (github.event.pull_request.head.repo.full_name) + required: true + head-sha: + description: Fork head SHA (github.event.pull_request.head.sha) + required: true + repository: + description: This repository (github.repository) + required: true + +runs: + using: composite + steps: + - name: Checkout base branch code (trusted) + uses: actions/checkout@v6 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.base-ref }} + fetch-depth: 0 + sparse-checkout: .github + sparse-checkout-cone-mode: false + + - name: Fetch base branch + shell: bash + run: git fetch origin ${{ inputs.base-ref }} + + - name: Save trusted code before fork checkout + shell: bash + run: cp -r .github /tmp/trusted-dotgithub + + - name: Checkout PR plugins (untrusted content only) + uses: actions/checkout@v6 + with: + repository: ${{ inputs.head-repo }} + ref: ${{ inputs.head-sha }} + fetch-depth: 0 + sparse-checkout: plugins + sparse-checkout-cone-mode: false + clean: false + # Fork content is only read as data (sparse plugins/), never executed; + # trusted code is saved/restored around this step. + allow-unsafe-pr-checkout: true + + - name: Restore trusted code + shell: bash + run: | + rm -rf .github + cp -r /tmp/trusted-dotgithub .github + + - name: Re-fetch base branch refs + shell: bash + run: git fetch https://github.com/${{ inputs.repository }} +${{ inputs.base-ref }}:refs/remotes/origin/${{ inputs.base-ref }} diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md new file mode 100644 index 00000000..3410b6fa --- /dev/null +++ b/.github/pluginctl/README.md @@ -0,0 +1,79 @@ +# pluginctl + +The automation CLI behind the plugin-registry workflows. Every piece of logic +that used to live in inline YAML or `.github/scripts/*.sh` now lives here as a +tested Python package, so the workflows are thin wiring and the behavior is +unit-testable. + +Standard library only. `git`, `gh`, `gpg`, and `clamscan` come from the CI +runner; there is no build step. + +## Layout + +``` +src/pluginctl/ + cli.py argparse dispatch (one subcommand per verb) + core/ shared infra: actions, git, gh, jsonio, models, version + validate/ PR-time pipeline: detect, title, labels, gate, sarif, + langs, clamav, plugin (per-plugin checks), report + publish/ releases pipeline: manifest, zips, cleanup, readmes, run, yank + integrations/ webhooks, external_readme, automerge +tests/ pytest suite (pure logic + golden outputs) +fixtures/ sample SARIF and other test inputs +``` + +## Develop + +```bash +cd .github/pluginctl +python -m pytest # run the suite +``` + +Run a command locally against a real checkout (cwd must be the repo root so +`plugins/` resolves): + +```bash +PYTHONPATH=.github/pluginctl/src python -m pluginctl \ + validate --plugin --author --base-ref main --out /tmp/frag.md +``` + +## Command map (old to new) + +| Old | New | +|---|---| +| `validate/detect-changes.sh` + inline blacklist | `pluginctl detect` | +| inline `validate-title` job | `pluginctl check-title` | +| inline `label-pr` job | `pluginctl label` | +| inline CodeQL SARIF jq (x3) | `pluginctl sarif` | +| inline CodeQL language detection | `pluginctl detect-langs` | +| inline ClamAV status/table | `pluginctl clamav-report` | +| `validate/validate.sh` | `pluginctl validate` | +| `validate/report.sh` | `pluginctl report` | +| inline gate ladder | `pluginctl gate` | +| `publish/run.sh` (+ chained scripts) | `pluginctl publish` | +| `publish/yank-version.sh` | `pluginctl yank` | +| inline `auto-merge-updates` | `pluginctl automerge` | +| inline `update-external-readme` | `pluginctl external-readme` | +| (new) signed events | `pluginctl webhook` | + +## How the workflows wire in + +- `.github/actions/gh-app-token` and `.github/actions/trusted-checkout` are + composite actions (reusable *steps*). +- `.github/workflows/_codeql-scan.yml` and `_clamav-scan.yml` are reusable + *workflows* (whole jobs), called by `validate-plugin.yml`. They must live in + `.github/workflows/` per GitHub; the `_` prefix marks them as internal. +- Every `python -m pluginctl` step sets `PYTHONPATH=.github/pluginctl/src`. In + the CodeQL/ClamAV jobs the package is loaded from a separate base-branch + checkout (`_trusted/`) so fork code from the PR merge tree is never executed. + +## Parity notes for maintainers + +- User-facing output (PR comments, manifests, READMEs) must stay byte-identical + to the old shell. `core/jsonio.py` reproduces `jq -c` exactly, and the + markdown renderers are ported line for line, guarded by golden tests. PR-comment + strings are copied verbatim from the old scripts, so match them exactly rather + than reformatting. +- `publish/readmes.py` intentionally corrects one old bug: deprecated plugins' + version count now comes from the manifest's `versions[]` instead of the + removed `zips/` path (which always rendered 0). diff --git a/.github/pluginctl/fixtures/sample.sarif b/.github/pluginctl/fixtures/sample.sarif new file mode 100644 index 00000000..77b431bf --- /dev/null +++ b/.github/pluginctl/fixtures/sample.sarif @@ -0,0 +1,43 @@ +{ + "runs": [ + { + "tool": { + "driver": { + "rules": [ + {"id": "py/sql-injection", "properties": {"security-severity": "8.8"}}, + {"id": "py/weak-hash", "properties": {"security-severity": "6.5"}}, + {"id": "py/clear-text-logging", "properties": {"security-severity": "5.0"}}, + {"id": "py/unused-import", "properties": {}} + ] + }, + "extensions": [ + {"rules": [ + {"id": "py/weak-hash", "properties": {"security-severity": "6.9"}} + ]} + ] + }, + "results": [ + { + "ruleId": "py/sql-injection", + "message": {"text": "This SQL query depends on a [user-provided value](123)."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/main.py"}, "region": {"startLine": 42}}}] + }, + { + "ruleId": "py/weak-hash", + "message": {"text": "Weak hash used at https://example.com/#4 www.foo"}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/util.py"}, "region": {"startLine": 7}}}] + }, + { + "ruleId": "py/clear-text-logging", + "message": {"text": "Sensitive data logged."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/log.py"}, "region": {"startLine": 3}}}] + }, + { + "ruleId": "py/unused-import", + "message": {"text": "Unused import."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/x.py"}, "region": {"startLine": 1}}}] + } + ] + } + ] +} diff --git a/.github/pluginctl/pyproject.toml b/.github/pluginctl/pyproject.toml new file mode 100644 index 00000000..3f27a436 --- /dev/null +++ b/.github/pluginctl/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pluginctl" +version = "2.0.0" +description = "Dispatcharr plugin registry automation CLI (validate / publish / yank / webhooks)" +requires-python = ">=3.10" +# Standard-library only. git / gh / gpg / clamscan are provided by the CI runner. +dependencies = [] + +[project.optional-dependencies] +test = ["pytest>=7.0"] + +[project.scripts] +pluginctl = "pluginctl.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["pluginctl*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/.github/pluginctl/src/pluginctl/__init__.py b/.github/pluginctl/src/pluginctl/__init__.py new file mode 100644 index 00000000..4718b949 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/__init__.py @@ -0,0 +1,3 @@ +"""pluginctl - Dispatcharr plugin registry automation CLI (V2).""" + +__version__ = "2.0.0" diff --git a/.github/pluginctl/src/pluginctl/__main__.py b/.github/pluginctl/src/pluginctl/__main__.py new file mode 100644 index 00000000..c1ba5622 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/__main__.py @@ -0,0 +1,6 @@ +"""Enable ``python -m pluginctl``.""" + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py new file mode 100644 index 00000000..1e7a58e8 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -0,0 +1,270 @@ +"""argparse dispatch for ``pluginctl`` / ``python -m pluginctl``. + +Each subcommand is a thin wrapper that reads flags (and falls back to the same +environment variables the workflows already set) and delegates to a handler +module. Handlers are imported lazily so a single missing optional dependency or +runtime tool never breaks unrelated commands. +""" + +from __future__ import annotations + +import argparse +import json +import os +from typing import Optional + + +def _matrix(value: Optional[str]) -> list[str]: + if not value: + return [] + try: + parsed = json.loads(value) + return [str(x) for x in parsed] if isinstance(parsed, list) else [] + except json.JSONDecodeError: + return [] + + +def _bool(value: Optional[str]) -> bool: + return str(value).lower() == "true" + + +def _env(name: str, default: str = "") -> str: + return os.environ.get(name, default) + + +# --------------------------------------------------------------------------- # +# Handlers +# --------------------------------------------------------------------------- # +def cmd_detect(args) -> int: + from .validate import detect + return detect.run( + pr_author=args.author, + base_ref=args.base_ref, + head_ref=args.head_ref or "", + author_blacklist=_env("AUTHOR_BLACKLIST"), + plugin_blacklist=_env("PLUGIN_BLACKLIST"), + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_check_title(args) -> int: + from .core import actions + from .validate import title + result = title.check_title(args.title, args.author, args.plugin_count, _matrix(args.matrix)) + actions.set_output("title_valid", "true" if result.valid else "false") + actions.set_output("title_feedback", result.feedback) + actions.set_output("title_suggestion", result.suggestion) + if not result.valid: + actions.error("PR title does not match the required format. See CONTRIBUTING.md for details.") + return 1 + return 0 + + +def cmd_label(args) -> int: + from .validate import labels + return labels.run( + pr_number=args.pr, + has_new_plugin=_bool(args.has_new_plugin), + has_updated_plugin=_bool(args.has_updated_plugin), + outside_files=args.outside_files or "", + outside_violation=_bool(args.outside_violation), + close_pr=_bool(args.close_pr), + ) + + +def cmd_gate(args) -> int: + from .core import actions + from .validate import gate + result = gate.evaluate( + detect_result=_env("DETECT_RESULT"), + close_pr=_env("CLOSE_PR"), + skip_validation=_env("SKIP_VALIDATION"), + outside_violation=_env("OUTSIDE_VIOLATION"), + title_result=_env("TITLE_RESULT"), + codeql_result=_env("CODEQL_RESULT"), + codeql_status=_env("CODEQL_STATUS"), + clamav_result=_env("CLAMAV_RESULT"), + clamav_status=_env("CLAMAV_STATUS"), + validate_result=_env("VALIDATE_RESULT"), + report_result=_env("REPORT_RESULT"), + ) + if result.ok: + actions.log(result.message) + return 0 + actions.error(result.message) + return 1 + + +def cmd_sarif(args) -> int: + from .validate import sarif + return sarif.run( + results_dir=args.results_dir, + repo=args.repo or _env("GITHUB_REPOSITORY"), + sha=args.sha, + matrix=_matrix(args.matrix), + analyze_outcome=args.analyze_outcome, + languages_found=_bool(args.found), + languages=args.languages or "", + unscanned_langs=args.unscanned_langs or "", + job_status=_env("CODEQL_ACTION_JOB_STATUS"), + ) + + +def cmd_validate(args) -> int: + from .validate import plugin as validate + return validate.run( + plugin_name=args.plugin, + pr_author=args.author, + base_ref=args.base_ref, + output_file=args.out, + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_report(args) -> int: + from .validate import report + return report.run( + pr_number=args.pr, + pr_author=args.author, + plugin_count=args.plugin_count, + close_pr=_bool(args.close_pr), + fragments_dir=args.fragments_dir, + repo=args.repo or _env("GITHUB_REPOSITORY"), + ) + + +def cmd_detect_langs(args) -> int: + from .validate import langs + return langs.run(root=args.root) + + +def cmd_clamav_report(args) -> int: + from .validate import clamav + return clamav.run(output_file=args.output, scan_exit=args.scan_exit, + findings_out=args.findings_out) + + +def cmd_webhook(args) -> int: + from .integrations import webhooks + data = json.loads(args.data) if args.data else {} + sent = webhooks.emit(args.event, data) + return 0 if sent or True else 1 # emission failures never fail the pipeline + + +def cmd_publish(args) -> int: + from .publish import run as publish_run + return publish_run.run(source_branch=args.source_branch) + + +def cmd_yank(args) -> int: + from .publish import yank + return yank.run() + + +def cmd_automerge(args) -> int: + from .integrations import automerge + return automerge.run(head_sha=args.head_sha or _env("HEAD_SHA")) + + +def cmd_external_readme(args) -> int: + from .integrations import external_readme + return external_readme.run() + + +# --------------------------------------------------------------------------- # +# Parser +# --------------------------------------------------------------------------- # +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="pluginctl", description="Dispatcharr plugin registry automation") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("detect", help="Detect changed plugins and build the matrix") + p.add_argument("--author", required=True) + p.add_argument("--base-ref", required=True) + p.add_argument("--head-ref", default="") + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_detect) + + p = sub.add_parser("check-title", help="Validate the PR title format") + p.add_argument("--title", required=True) + p.add_argument("--author", required=True) + p.add_argument("--plugin-count", type=int, default=0) + p.add_argument("--matrix", default="[]") + p.set_defaults(func=cmd_check_title) + + p = sub.add_parser("label", help="Reconcile PR classification labels") + p.add_argument("--pr", required=True) + p.add_argument("--has-new-plugin", default="false") + p.add_argument("--has-updated-plugin", default="false") + p.add_argument("--outside-files", default="") + p.add_argument("--outside-violation", default="false") + p.add_argument("--close-pr", default="false") + p.set_defaults(func=cmd_label) + + p = sub.add_parser("gate", help="Evaluate the Plugin PR Check gate (env-driven)") + p.set_defaults(func=cmd_gate) + + p = sub.add_parser("sarif", help="Parse CodeQL SARIF: counts, tables, status") + p.add_argument("--results-dir", default="sarif-results") + p.add_argument("--repo", default="") + p.add_argument("--sha", default="") + p.add_argument("--matrix", default="[]") + p.add_argument("--analyze-outcome", default="") + p.add_argument("--found", default="false") + p.add_argument("--languages", default="") + p.add_argument("--unscanned-langs", default="") + p.set_defaults(func=cmd_sarif) + + p = sub.add_parser("validate", help="Validate one plugin -> markdown fragment") + p.add_argument("--plugin", required=True) + p.add_argument("--author", required=True) + p.add_argument("--base-ref", required=True) + p.add_argument("--out", required=True) + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_validate) + + p = sub.add_parser("report", help="Aggregate fragments, post PR comment") + p.add_argument("--pr", required=True) + p.add_argument("--author", required=True) + p.add_argument("--plugin-count", required=True) + p.add_argument("--close-pr", default="false") + p.add_argument("--fragments-dir", default=".") + p.add_argument("--repo", default="") + p.set_defaults(func=cmd_report) + + p = sub.add_parser("detect-langs", help="Detect CodeQL languages in changed plugins") + p.add_argument("--root", default="plugins") + p.set_defaults(func=cmd_detect_langs) + + p = sub.add_parser("clamav-report", help="Parse clamscan output: status + findings table") + p.add_argument("--output", default="clamav-output.txt") + p.add_argument("--scan-exit", type=int, default=0) + p.add_argument("--findings-out", default="clamav-findings.md") + p.set_defaults(func=cmd_clamav_report) + + p = sub.add_parser("webhook", help="Emit a signed webhook event") + p.add_argument("--event", required=True) + p.add_argument("--data", default="{}") + p.set_defaults(func=cmd_webhook) + + p = sub.add_parser("publish", help="Publish plugins to the releases branch") + p.add_argument("source_branch") + p.set_defaults(func=cmd_publish) + + p = sub.add_parser("yank", help="Yank a plugin version (env-driven)") + p.set_defaults(func=cmd_yank) + + p = sub.add_parser("automerge", help="Auto-merge a pure plugin-update PR") + p.add_argument("--head-sha", default="") + p.set_defaults(func=cmd_automerge) + + p = sub.add_parser("external-readme", help="Open/update the external README PR (env-driven)") + p.set_defaults(func=cmd_external_readme) + + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + return args.func(args) diff --git a/.github/pluginctl/src/pluginctl/core/__init__.py b/.github/pluginctl/src/pluginctl/core/__init__.py new file mode 100644 index 00000000..b9f67b6a --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/__init__.py @@ -0,0 +1 @@ +"""Shared infrastructure: JSON, versioning, git/gh wrappers, Actions plumbing.""" diff --git a/.github/pluginctl/src/pluginctl/core/actions.py b/.github/pluginctl/src/pluginctl/core/actions.py new file mode 100644 index 00000000..3ba14f15 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/actions.py @@ -0,0 +1,66 @@ +"""GitHub Actions plumbing: GITHUB_OUTPUT, step summary, workflow commands. + +Thin, side-effecting helpers that write to the files/streams the runner reads. +Kept tiny so command modules stay testable (they build values; these emit them). +""" + +from __future__ import annotations + +import os +import sys +import uuid + + +def set_output(name: str, value: str) -> None: + """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe). + + Values containing a newline use the heredoc form GitHub documents, with a + random delimiter to avoid collisions with the payload. + """ + path = os.environ.get("GITHUB_OUTPUT") + if not path: + # Local/dry-run: echo to stdout the same way the shell scripts did. + print(f"{name}={value}") + return + with open(path, "a", encoding="utf-8") as fh: + if "\n" in value: + delim = f"ghadelim_{uuid.uuid4().hex}" + fh.write(f"{name}<<{delim}\n{value}\n{delim}\n") + else: + fh.write(f"{name}={value}\n") + + +def set_outputs(**kwargs: str) -> None: + for name, value in kwargs.items(): + set_output(name, "" if value is None else str(value)) + + +def step_summary(markdown: str) -> None: + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + print(markdown) + return + with open(path, "a", encoding="utf-8") as fh: + fh.write(markdown) + if not markdown.endswith("\n"): + fh.write("\n") + + +def error(message: str) -> None: + print(f"::error::{message}") + + +def warning(message: str) -> None: + print(f"::warning::{message}") + + +def notice(message: str) -> None: + print(f"::notice::{message}") + + +def log(message: str) -> None: + print(message) + + +def eprint(message: str) -> None: + print(message, file=sys.stderr) diff --git a/.github/pluginctl/src/pluginctl/core/gh.py b/.github/pluginctl/src/pluginctl/core/gh.py new file mode 100644 index 00000000..a3d35c26 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/gh.py @@ -0,0 +1,183 @@ +"""Thin wrapper over the ``gh`` CLI and the GitHub REST/GraphQL surface we use. + +Every call shells out to ``gh`` (already authenticated on the runner via +``GH_TOKEN``), matching how the V1 scripts invoked it. Helpers return parsed +values and swallow errors the same way the ``|| true`` / ``2>/dev/null`` idioms +did in Bash, so callers keep the same permissive behavior. +""" + +from __future__ import annotations + +import json +import subprocess +from typing import Any, Optional + + +def _run(args: list[str], check: bool = False, input_text: Optional[str] = None) -> subprocess.CompletedProcess: + return subprocess.run( + ["gh", *args], + check=check, + capture_output=True, + text=True, + input=input_text, + ) + + +def api(endpoint: str, jq: Optional[str] = None, method: Optional[str] = None, + fields: Optional[dict[str, str]] = None) -> Optional[str]: + """``gh api `` returning stripped stdout, or None on failure.""" + args = ["api", endpoint] + if method: + args += ["-X", method] + if jq: + args += ["--jq", jq] + for k, v in (fields or {}).items(): + args += ["-f", f"{k}={v}"] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def collaborator_permission(owner: str, repo: str, user: str) -> str: + """Return the collaborator permission string, or ``none`` (matches Bash).""" + perm = api(f"repos/{owner}/{repo}/collaborators/{user}/permission", + jq=".permission") + return perm if perm else "none" + + +def has_write_access(owner: str, repo: str, user: str) -> bool: + """True when the user has admin/maintain/write, mirroring has_write_access().""" + return collaborator_permission(owner, repo, user) in ("admin", "maintain", "write") + + +def graphql(query: str, f_fields: Optional[dict[str, str]] = None, + F_fields: Optional[dict[str, str]] = None, jq: Optional[str] = None) -> Optional[str]: + args = ["api", "graphql", "-f", f"query={query}"] + for k, v in (f_fields or {}).items(): + args += ["-f", f"{k}={v}"] + for k, v in (F_fields or {}).items(): + args += ["-F", f"{k}={v}"] + if jq: + args += ["--jq", jq] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def pr_comment(pr_number: str, body: str) -> int: + """``gh pr comment``: returns the exit code (report.sh keys off it).""" + return _run(["pr", "comment", str(pr_number), "--body", body]).returncode + + +def pr_close(pr_number: str) -> int: + return _run(["pr", "close", str(pr_number)]).returncode + + +def pr_comment_repo(pr_number: str, repo: str, body: str) -> int: + return _run(["pr", "comment", str(pr_number), "--repo", repo, "--body", body]).returncode + + +def pr_edit_label(pr_number: str, label: str, add: bool) -> None: + flag = "--add-label" if add else "--remove-label" + _run(["pr", "edit", str(pr_number), flag, label]) + + +def label_create(name: str, color: str, description: str, repo: str) -> None: + _run(["label", "create", name, "--color", color, + "--description", description, "--repo", repo]) + + +def pr_view_json(pr_number: str, fields: str, jq: Optional[str] = None, + repo: Optional[str] = None) -> Optional[str]: + args = ["pr", "view", str(pr_number), "--json", fields] + if jq: + args += ["--jq", jq] + if repo: + args += ["--repo", repo] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def loads(text: Optional[str]) -> Any: + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +# ---- GitHub Releases helpers (publish / yank / cleanup) --------------------- +def release_exists(tag: str, repo: str) -> bool: + return _run(["release", "view", tag, "--repo", repo]).returncode == 0 + + +def release_list_tags(repo: str, limit: int = 500) -> list[str]: + proc = _run(["release", "list", "--repo", repo, "--json", "tagName", "--limit", str(limit)]) + if proc.returncode != 0: + return [] + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError: + return [] + return [entry.get("tagName", "") for entry in data if entry.get("tagName")] + + +def release_create(tag: str, repo: str, title: str, notes: str, asset: str) -> int: + return _run(["release", "create", tag, "--repo", repo, "--title", title, + "--notes", notes, asset]).returncode + + +def release_delete(tag: str, repo: str, cleanup_tag: bool = True) -> int: + args = ["release", "delete", tag, "--repo", repo, "--yes"] + if cleanup_tag: + args.append("--cleanup-tag") + return _run(args).returncode + + +def pr_list(repo: str, state: str, fields: str, jq: Optional[str] = None) -> Optional[str]: + args = ["pr", "list", "--repo", repo, "--state", state, "--json", fields] + if jq: + args += ["--jq", jq] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def pr_merge_squash(pr: str, repo: str, delete_branch: bool = False) -> int: + return _run(["pr", "merge", str(pr), "--repo", repo, "--squash", + f"--delete-branch={str(delete_branch).lower()}"]).returncode + + +def issue_comment(issue: str, repo: str, body: str) -> int: + return _run(["issue", "comment", str(issue), "--repo", repo, "--body", body]).returncode + + +def issue_close(issue: str, repo: str, reason: str = "completed") -> int: + return _run(["issue", "close", str(issue), "--repo", repo, "--reason", reason]).returncode + + +def pr_create(repo: str, base: str, head: str, title: str, body: str, + labels: Optional[list[str]] = None) -> Optional[str]: + args = ["pr", "create", "--repo", repo, "--base", base, "--head", head, + "--title", title, "--body", body] + for label in (labels or []): + args += ["--label", label] + proc = _run(args) + if proc.returncode != 0: + return None + return proc.stdout.strip() + + +def commit_pull_number(repo: str, sha: str) -> tuple[Optional[str], Optional[str]]: + """The first PR associated with a commit, as (number, html_url) or (None, None).""" + out = api(f"repos/{repo}/commits/{sha}/pulls", + jq='.[0] | {number: .number, url: .html_url}') + data = loads(out) or {} + number = data.get("number") + return (str(number) if number else None, data.get("url")) diff --git a/.github/pluginctl/src/pluginctl/core/git.py b/.github/pluginctl/src/pluginctl/core/git.py new file mode 100644 index 00000000..d347f587 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/git.py @@ -0,0 +1,80 @@ +"""Thin wrappers over the local ``git`` CLI used by detect / validate / publish.""" + +from __future__ import annotations + +import subprocess +from typing import Optional + + +def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], + check=check, + capture_output=True, + text=True, + ) + + +def merge_base(a: str, b: str) -> str: + return _run(["merge-base", a, b]).stdout.strip() + + +def diff_name_only(base: str, head: str = "HEAD", paths: Optional[list[str]] = None) -> list[str]: + args = ["diff", "--name-only", base, head] + if paths: + args += ["--", *paths] + out = _run(args).stdout + return [line for line in out.splitlines() if line] + + +def diff_name_only_range(range_spec: str, paths: Optional[list[str]] = None) -> list[str]: + """``git diff --name-only [-- paths]`` where range is e.g. ``a...HEAD``.""" + args = ["diff", "--name-only", range_spec] + if paths: + args += ["--", *paths] + out = _run(args).stdout + return [line for line in out.splitlines() if line] + + +def show(ref_path: str) -> Optional[str]: + """``git show :`` returning None when the object does not exist.""" + proc = _run(["show", ref_path], check=False) + if proc.returncode != 0: + return None + return proc.stdout + + +def object_exists(ref_path: str) -> bool: + return _run(["show", ref_path], check=False).returncode == 0 + + +def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: + """``git log -1 --format= [-- ]`` (None if empty/failure).""" + args = ["log", "-1", f"--format={fmt}", ref] + if path is not None: + args += ["--", path] + proc = _run(args, check=False) + if proc.returncode != 0: + return None + out = proc.stdout.strip() + return out or None + + +def rev_parse(rev: str, short: bool = False) -> str: + args = ["rev-parse"] + if short: + args.append("--short") + args.append(rev) + return _run(args).stdout.strip() + + +def fetch(*args: str) -> None: + _run(["fetch", *args], check=False) + + +def sparse_checkout_add(path: str) -> bool: + return _run(["sparse-checkout", "add", path], check=False).returncode == 0 + + +def checkout(*args: str) -> bool: + return _run(["checkout", *args], check=False).returncode == 0 diff --git a/.github/pluginctl/src/pluginctl/core/jsonio.py b/.github/pluginctl/src/pluginctl/core/jsonio.py new file mode 100644 index 00000000..d9a1ff28 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/jsonio.py @@ -0,0 +1,38 @@ +"""Canonical JSON helpers that reproduce the exact byte output of the V1 jq programs. + +The V1 publish pipeline builds manifests with `jq` and emits them with `jq -c`. +To keep the published `releases`-branch output byte-identical, every manifest +object here is built as an insertion-ordered ``dict`` in the same key order as the +corresponding jq program, nulls are dropped the same way jq's +``with_entries(select(.value != null))`` does, and the compact form is produced +with the same separators jq uses. +""" + +from __future__ import annotations + +import json +from typing import Any + + +def dumps(obj: Any) -> str: + """Compact JSON, byte-equivalent to ``jq -c '.'``. + + jq -c uses ``,``/``:`` separators, emits UTF-8 (no ``\\uXXXX`` escaping of + non-ASCII), and does not escape ``/``. ``json.dumps`` with these options + matches on all three counts. + """ + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False) + + +def drop_none(obj: dict) -> dict: + """Reproduce ``with_entries(select(.value != null))`` for a single object. + + Only top-level ``None`` values are removed, insertion order preserved, + matching the jq idiom used throughout ``generate-manifest.sh``. + """ + return {k: v for k, v in obj.items() if v is not None} + + +def compact(obj: Any) -> str: + """Alias for :func:`dumps` for call-site readability.""" + return dumps(obj) diff --git a/.github/pluginctl/src/pluginctl/core/models.py b/.github/pluginctl/src/pluginctl/core/models.py new file mode 100644 index 00000000..d88625b3 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/models.py @@ -0,0 +1,97 @@ +"""Typed view over a ``plugin.json`` document. + +Wraps the parsed dict and exposes the handful of accessors the validate/publish +logic needs, with the same ``// ""`` / ``// false`` defaulting the jq programs +used. ``raw`` stays available for the manifest builders that must preserve the +original key set and ordering. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass +class Plugin: + raw: dict = field(default_factory=dict) + + # ---- constructors ------------------------------------------------------- + @classmethod + def from_json_text(cls, text: str) -> "Plugin": + return cls(raw=json.loads(text)) + + @classmethod + def from_file(cls, path: str) -> "Plugin": + with open(path, encoding="utf-8") as fh: + return cls(raw=json.load(fh)) + + # ---- scalar accessors (jq `// ""` semantics) ---------------------------- + def str_field(self, key: str, default: str = "") -> str: + val = self.raw.get(key) + return default if val is None else str(val) + + @property + def name(self) -> str: + return self.str_field("name") + + @property + def version(self) -> str: + return self.str_field("version") + + @property + def description(self) -> str: + return self.str_field("description") + + @property + def author(self) -> str: + return self.str_field("author") + + @property + def license(self) -> str: + return self.str_field("license") + + @property + def repo_url(self) -> str: + return self.str_field("repo_url") + + @property + def discord_thread(self) -> str: + return self.str_field("discord_thread") + + @property + def source_type(self) -> str: + return self.str_field("source_type", "local") + + @property + def source_url(self) -> str: + return self.str_field("source_url") + + @property + def min_dispatcharr_version(self) -> str: + return self.str_field("min_dispatcharr_version") + + @property + def max_dispatcharr_version(self) -> str: + return self.str_field("max_dispatcharr_version") + + @property + def deprecated(self) -> bool: + return self.raw.get("deprecated") is True + + @property + def unlisted(self) -> bool: + return self.raw.get("unlisted") is True + + @property + def maintainers(self) -> list[str]: + """``[.maintainers[]?]``: always a list, dropping non-list values.""" + val = self.raw.get("maintainers") + return list(val) if isinstance(val, list) else [] + + def maintainers_joined(self, sep: str = " ") -> str: + return sep.join(str(m) for m in self.maintainers) + + def get(self, key: str, default: Any = None) -> Any: + return self.raw.get(key, default) diff --git a/.github/pluginctl/src/pluginctl/core/version.py b/.github/pluginctl/src/pluginctl/core/version.py new file mode 100644 index 00000000..f279b8fc --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/version.py @@ -0,0 +1,89 @@ +"""Version parsing / comparison helpers ported from the Bash validate + publish scripts. + +These reproduce the exact acceptance rules of the shell functions: + validate_semver -> ^[0-9]+\\.[0-9]+\\.[0-9]+$ + validate_dispatcharr_version -> ^v?[0-9]+\\.[0-9]+\\.[0-9]+$ + version_greater_than -> field-wise major/minor/patch comparison +and the `sort -V -r` ordering used to pick newest release tags. +""" + +from __future__ import annotations + +import re +from functools import cmp_to_key + +_SEMVER_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+$") +_DA_VERSION_RE = re.compile(r"^v?[0-9]+\.[0-9]+\.[0-9]+$") + + +def is_semver(version: str) -> bool: + """True for strict ``X.Y.Z`` (matches validate_semver == 1).""" + return bool(_SEMVER_RE.match(version or "")) + + +def is_dispatcharr_version(version: str) -> bool: + """True for ``X.Y.Z`` or ``vX.Y.Z`` (matches validate_dispatcharr_version == 1).""" + return bool(_DA_VERSION_RE.match(version or "")) + + +def version_greater_than(new_version: str, old_version: str) -> bool: + """Return True when new > old, comparing major/minor/patch as integers. + + Mirrors the Bash ``version_greater_than`` exactly: it reads three + dot-separated fields and compares them numerically; equal versions are not + "greater". + """ + def _parts(v: str) -> tuple[int, int, int]: + fields = (v or "").split(".") + # Bash `IFS='.' read -r A B C` leaves missing fields empty -> treated as 0 + nums = [] + for i in range(3): + try: + nums.append(int(fields[i])) + except (IndexError, ValueError): + nums.append(0) + return nums[0], nums[1], nums[2] + + nmaj, nmin, npat = _parts(new_version) + omaj, omin, opat = _parts(old_version) + if nmaj > omaj: + return True + if nmaj < omaj: + return False + if nmin > omin: + return True + if nmin < omin: + return False + if npat > opat: + return True + return False + + +def _version_sort_key(s: str): + """A key that orders like GNU ``sort -V`` for the version strings we handle. + + Release tags are stripped to bare versions such as ``1.2.3`` or ``1.2.3-1`` + (a migration retry suffix). ``sort -V`` treats each maximal run of digits as a + number and each run of non-digits lexically, so we tokenize accordingly. + """ + tokens: list = [] + for part in re.findall(r"\d+|\D+", s or ""): + if part.isdigit(): + tokens.append((1, int(part), "")) + else: + tokens.append((0, 0, part)) + return tokens + + +def _version_cmp(a: str, b: str) -> int: + ka, kb = _version_sort_key(a), _version_sort_key(b) + if ka < kb: + return -1 + if ka > kb: + return 1 + return 0 + + +def sort_versions_desc(versions: list[str]) -> list[str]: + """Sort bare version strings newest-first, like ``sort -V -r``.""" + return sorted(versions, key=cmp_to_key(_version_cmp), reverse=True) diff --git a/.github/pluginctl/src/pluginctl/integrations/__init__.py b/.github/pluginctl/src/pluginctl/integrations/__init__.py new file mode 100644 index 00000000..44101277 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/__init__.py @@ -0,0 +1 @@ +"""Outward integrations: signed webhooks, external README PR, auto-merge.""" diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py new file mode 100644 index 00000000..bef26116 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -0,0 +1,111 @@ +"""Auto-merge pure plugin-update PRs (port of auto-merge-updates.yml inline logic). + +Only squash-merges a PR that carries the ``Plugin Update`` label, none of the +blocking labels, is cleanly mergeable, and independently re-verifies from the diff +that it touches only existing plugins under ``plugins/``. Every rejection is a +notice, never a failure. The workflow already gates on the GitHub App token being +configured (a GITHUB_TOKEN merge would not cascade into publish), so this handler +assumes it is invoked only in that case. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from dataclasses import dataclass + +from ..core import actions, gh + +BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE") + + +def _git(*args: str, check: bool = False) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +@dataclass +class LabelDecision: + ok: bool + reason: str = "" + + +def evaluate_labels(labels: list[str], mergeable: str) -> LabelDecision: + """Require Plugin Update, forbid the blocking labels, require MERGEABLE.""" + label_set = set(labels) + if "Plugin Update" not in label_set: + return LabelDecision(False, "Missing 'Plugin Update' label") + for blocked in BLOCKING_LABELS: + if blocked in label_set: + return LabelDecision(False, f"Has '{blocked}' label") + if mergeable != "MERGEABLE": + return LabelDecision(False, f"PR not cleanly mergeable (mergeable={mergeable})") + return LabelDecision(True) + + +def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: + """Re-derive from the diff that the PR only updates existing plugins.""" + if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}").returncode != 0: + actions.notice(f"Could not fetch merge ref for PR #{pr_number} - skipping.") + return False + _git("checkout", "-q", f"pr-merge-{pr_number}") + merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD").stdout.strip() + changed = _git("diff", "--name-only", merge_base, "HEAD").stdout.splitlines() + + outside = [f for f in changed if not f.startswith("plugins/")] + if outside: + actions.notice(f"Changes outside plugins/ detected - skipping. Files: {' '.join(outside)}") + return False + + slugs = sorted({f.split("/")[1] for f in changed + if f.startswith("plugins/") and len(f.split("/")) > 1}) + if not slugs: + actions.notice("No plugin changes detected in diff - skipping.") + return False + + for slug in slugs: + if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json").returncode != 0: + actions.notice(f"Plugin '{slug}' does not exist on {base_branch} - this is a new " + "plugin, not an update. Skipping.") + return False + return True + + +def run(head_sha: str) -> int: + repo = os.environ.get("GH_REPO") or os.environ.get("GITHUB_REPOSITORY", "") + + matches_json = gh.pr_list(repo, "open", "number,headRefOid,isDraft") or "[]" + try: + matches = [pr for pr in json.loads(matches_json) if pr.get("headRefOid") == head_sha] + except json.JSONDecodeError: + matches = [] + if len(matches) != 1: + actions.notice(f"Found {len(matches)} open PR(s) matching head SHA {head_sha} - " + "skipping (expected exactly 1).") + return 0 + pr = matches[0] + pr_number = str(pr.get("number")) + if pr.get("isDraft") is True: + actions.notice(f"PR #{pr_number} is a draft - skipping.") + return 0 + + info = gh.pr_view_json(pr_number, "labels,mergeable,mergeStateStatus", repo=repo) + data = gh.loads(info) or {} + labels = [l.get("name") for l in (data.get("labels") or [])] + mergeable = data.get("mergeable") or "" + actions.log(f"Labels: {','.join(labels)}") + actions.log(f"Mergeable: {mergeable}") + + decision = evaluate_labels(labels, mergeable) + if not decision.ok: + actions.notice(f"{decision.reason} - skipping.") + return 0 + + if not reverify_is_pure_update(pr_number): + return 0 + + gh.pr_merge_squash(pr_number, repo, delete_branch=False) + actions.log(f"Auto-merged PR #{pr_number}") + from . import webhooks + webhooks.emit("pr.auto_merged", webhooks.pr_auto_merged(pr_number, [])) + return 0 diff --git a/.github/pluginctl/src/pluginctl/integrations/external_readme.py b/.github/pluginctl/src/pluginctl/integrations/external_readme.py new file mode 100644 index 00000000..eb95ed98 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/external_readme.py @@ -0,0 +1,157 @@ +"""Open/update the plugin-listing PR in an external repo (port of update-external-readme.yml). + +Reads the releases-branch README, strips the releases-only preamble, rewrites its +relative links to absolute releases-branch URLs, then creates or updates a PR in +the configured target repo. The text-transform helpers are pure and unit-tested; +the PR plumbing uses the GitHub Contents/PR APIs. Skips when the only change is the +timestamp line. +""" + +from __future__ import annotations + +import base64 +import os +import re +import subprocess +from typing import Optional + +from ..core import actions, gh + +BRANCH = "auto/dispatcharr-plugin-readme" +_SOURCE_COMMIT_RE = re.compile(r"Source commit: ([0-9a-f]+)") +_PLUGIN_LINE_RE = re.compile(r"^- [a-z0-9].*@") + + +def extract_metadata(commit_msg: str) -> tuple[str, str]: + """Return (source_commit, plugin_list_block) from the releases-branch commit.""" + m = _SOURCE_COMMIT_RE.search(commit_msg) + source_commit = m.group(1) if m else "" + plugin_lines = [ln for ln in commit_msg.splitlines() if _PLUGIN_LINE_RE.match(ln)] + return source_commit, "\n".join(plugin_lines) + + +def strip_preamble(readme: str, contrib_url: str) -> str: + """Keep the title + a contribute line, drop the intro/Quick Access, keep from Available Plugins.""" + out: list[str] = [] + found = False + for line in readme.splitlines(): + if line.startswith("# Plugin Releases"): + out.append(line) + out.append("") + out.append(f"Want to get your plugin added to this list? Check out the " + f"[plugin repository]({contrib_url}) to learn how to contribute.") + continue + if line.startswith("## Available Plugins"): + found = True + if found: + out.append(line) + return "\n".join(out) + ("\n" if readme.endswith("\n") else "") + + +def rewrite_links(text: str, base_url: str) -> str: + """Rewrite relative ``](./`` links to absolute releases-branch URLs.""" + return text.replace("](./", f"]({base_url}/") + + +def _strip_timestamp(text: str) -> str: + return "\n".join(ln for ln in text.splitlines() if not ln.startswith("*Last updated:")) + + +def run(releases_ref: str = "origin/releases") -> int: + """Read the releases README from ``releases_ref`` and sync it to the target repo. + + Sourcing from a git ref (rather than a working-tree checkout) lets the caller + keep the base branch checked out, where the composite actions and this package + live, while still reading the releases-branch content and its publish commit. + """ + target_repo = os.environ.get("TARGET_REPO", "") + target_path = os.environ.get("TARGET_PATH", "") + source_repo = os.environ.get("SOURCE_REPO") or os.environ.get("GITHUB_REPOSITORY", "") + reviewer = os.environ.get("PR_REVIEWER", "") + + show = subprocess.run(["git", "show", f"{releases_ref}:README.md"], + capture_output=True, text=True) + if show.returncode != 0: + actions.error("README.md not found on the releases branch - has the Publish Plugins workflow run yet?") + return 1 + readme = show.stdout + + commit_msg = subprocess.run(["git", "log", "-1", "--format=%B", releases_ref], + capture_output=True, text=True).stdout + source_commit, plugin_list = extract_metadata(commit_msg) + releases_commit = subprocess.run(["git", "rev-parse", "--short", releases_ref], + capture_output=True, text=True).stdout.strip() + + contrib_url = f"https://github.com/{source_repo}/blob/main/CONTRIBUTING.md" + readme = strip_preamble(readme, contrib_url) + readme = rewrite_links(readme, f"https://github.com/{source_repo}/tree/releases") + + default_branch = gh.api(f"repos/{target_repo}", jq=".default_branch") + if not default_branch: + actions.error(f"Could not resolve default branch for {target_repo}.") + return 1 + + if gh.api(f"repos/{target_repo}/branches/{BRANCH}") is None: + base_sha = gh.api(f"repos/{target_repo}/git/refs/heads/{default_branch}", jq=".object.sha") + gh.api(f"repos/{target_repo}/git/refs", method="POST", + fields={"ref": f"refs/heads/{BRANCH}", "sha": base_sha or ""}) + actions.log(f"Created branch {BRANCH}") + + existing = gh.api(f"repos/{target_repo}/contents/{target_path}?ref={BRANCH}") + file_sha = "" + if existing: + data = gh.loads(existing) or {} + file_sha = data.get("sha") or "" + try: + existing_content = base64.b64decode(data.get("content", "")).decode("utf-8") + except Exception: + existing_content = "" + if _strip_timestamp(readme) == _strip_timestamp(existing_content): + actions.log("README content unchanged (only timestamp differs) - skipping.") + actions.notice("External README not updated: only the timestamp line changed.") + return 0 + + put_fields = { + "message": f"chore: update plugin releases listing (source commit {source_commit})", + "content": base64.b64encode(readme.encode("utf-8")).decode("ascii"), + "branch": BRANCH, + } + if file_sha: + put_fields["sha"] = file_sha + gh.api(f"repos/{target_repo}/contents/{target_path}", method="PUT", fields=put_fields) + + summary = _build_summary(source_repo, source_commit, releases_commit, plugin_list) + existing_pr = gh.pr_list(target_repo, "open", "number", jq=".[0].number // empty") + if existing_pr: + gh.pr_comment_repo(existing_pr, target_repo, f"### Plugin README update\n\n{summary}") + actions.log(f"Added update comment to existing PR #{existing_pr}") + else: + body = (f"Automated update of the plugin releases listings generated from " + f"[`{source_repo}`](https://github.com/{source_repo}).\n\n{summary}") + labels = None + reviewers = [r.lstrip("@").replace(" ", "") for r in reviewer.split(",") if r.strip()] + _pr_create_with_reviewers(target_repo, BRANCH, default_branch, + "chore: update plugin releases listings", body, reviewers) + actions.log("Created new PR") + return 0 + + +def _build_summary(source_repo, source_commit, releases_commit, plugin_list) -> str: + lines = [ + f"**Source commit:** [`{source_commit}`](https://github.com/{source_repo}/commit/{source_commit})", + f"**Releases commit:** [`{releases_commit}`](https://github.com/{source_repo}/commit/{releases_commit})", + ] + if plugin_list: + lines.append("") + lines.append("**Plugins updated:**") + lines.extend(plugin_list.splitlines()) + return "\n".join(lines) + + +def _pr_create_with_reviewers(repo, head, base, title, body, reviewers) -> None: + args = ["gh", "pr", "create", "--repo", repo, "--head", head, "--base", base, + "--title", title, "--body", body] + for r in reviewers: + if r: + args += ["--reviewer", r] + subprocess.run(args, capture_output=True, text=True) diff --git a/.github/pluginctl/src/pluginctl/integrations/webhooks.py b/.github/pluginctl/src/pluginctl/integrations/webhooks.py new file mode 100644 index 00000000..1bd71e7d --- /dev/null +++ b/.github/pluginctl/src/pluginctl/integrations/webhooks.py @@ -0,0 +1,118 @@ +"""Signed webhook emitter (new capability). + +Emits a compact JSON event to ``WEBHOOK_URL`` signed with ``WEBHOOK_SECRET`` via +HMAC-SHA256 over the raw request body. Both settings are optional: with either +unset, :func:`emit` is a silent no-op. A delivery failure never raises, it logs +a ``::warning::``, so the pipeline is never failed by webhook problems. + +Header scheme (documented in docs/webhooks.md for the consuming bot): + X-PluginCtl-Event + X-PluginCtl-Delivery + X-PluginCtl-Signature sha256= +""" + +from __future__ import annotations + +import datetime +import hashlib +import hmac +import json +import os +import urllib.error +import urllib.request +import uuid +from typing import Any, Optional + +from ..core import actions + +USER_AGENT = "pluginctl-webhook/1" + + +def sign(secret: str, body: bytes) -> str: + """Return ``sha256=`` HMAC of ``body`` using ``secret`` (utf-8).""" + digest = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + return f"sha256={digest}" + + +def build_envelope(event: str, data: dict, repository: str, actor: str) -> dict: + return { + "event": event, + "delivered_at": datetime.datetime.now(datetime.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "repository": repository, + "actor": actor, + "data": data, + } + + +def emit(event: str, data: dict, *, repository: Optional[str] = None, + actor: Optional[str] = None, url: Optional[str] = None, + secret: Optional[str] = None, _opener=None) -> bool: + """Deliver an event. Returns True if sent, False if skipped or failed. + + Configuration falls back to the environment (``WEBHOOK_URL`` / + ``WEBHOOK_SECRET`` / ``GITHUB_REPOSITORY`` / ``GITHUB_ACTOR``) so callers can + pass nothing in CI. + """ + url = url if url is not None else os.environ.get("WEBHOOK_URL", "") + secret = secret if secret is not None else os.environ.get("WEBHOOK_SECRET", "") + repository = repository if repository is not None else os.environ.get("GITHUB_REPOSITORY", "") + actor = actor if actor is not None else os.environ.get("GITHUB_ACTOR", "") + + if not url or not secret: + return False + + envelope = build_envelope(event, data, repository, actor) + body = json.dumps(envelope, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + delivery = str(uuid.uuid4()) + request = urllib.request.Request( + url, + data=body, + method="POST", + headers={ + "Content-Type": "application/json", + "User-Agent": USER_AGENT, + "X-PluginCtl-Event": event, + "X-PluginCtl-Delivery": delivery, + "X-PluginCtl-Signature": sign(secret, body), + }, + ) + opener = _opener or urllib.request.urlopen + try: + with opener(request, timeout=15) as resp: + status = getattr(resp, "status", None) or resp.getcode() + if status and 200 <= int(status) < 300: + actions.log(f"Webhook '{event}' delivered ({delivery}, HTTP {status}).") + return True + actions.warning(f"Webhook '{event}' returned HTTP {status} - ignoring.") + return False + except (urllib.error.URLError, OSError, ValueError) as exc: # never fail the pipeline + actions.warning(f"Webhook '{event}' delivery failed: {exc}") + return False + + +# ---- typed event constructors (thin, so call sites stay readable) ----------- +def pr_validated(pr, author, result, plugins, checks) -> dict: + return {"pr": pr, "author": author, "result": result, + "plugins": plugins, "checks": checks} + + +def pr_closed_unauthorized(pr, author, reason) -> dict: + return {"pr": pr, "author": author, "reason": reason} + + +def pr_quarantined(pr, author, infected) -> dict: + return {"pr": pr, "author": author, "infected": infected} + + +def pr_auto_merged(pr, plugins) -> dict: + return {"pr": pr, "plugins": plugins} + + +def plugin_published(plugin, version, pr, actor) -> dict: + return {"plugin": plugin, "version": version, "pr": pr, "actor": actor} + + +def plugin_yanked(plugin, version, issue, rollback_pr) -> dict: + return {"plugin": plugin, "version": version, "issue": issue, + "rollback_pr": rollback_pr} diff --git a/.github/pluginctl/src/pluginctl/publish/__init__.py b/.github/pluginctl/src/pluginctl/publish/__init__.py new file mode 100644 index 00000000..7134aec5 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/__init__.py @@ -0,0 +1 @@ +"""Publish pipeline: build ZIPs + GitHub Releases, manifests, and READMEs.""" diff --git a/.github/pluginctl/src/pluginctl/publish/cleanup.py b/.github/pluginctl/src/pluginctl/publish/cleanup.py new file mode 100644 index 00000000..f090a242 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/cleanup.py @@ -0,0 +1,47 @@ +"""Remove releases for deleted plugins and prune old versions (port of cleanup.sh). + +Runs from the releases-branch checkout directory. +""" + +from __future__ import annotations + +import glob +import os +import shutil + +from ..core import actions, gh +from ..core.version import sort_versions_desc + + +def _versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: + prefix = f"{plugin_name}-" + latest = f"{plugin_name}-latest" + versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != latest] + return [f"{prefix}{v}" for v in sort_versions_desc(versions)] + + +def run(repository: str, max_versioned_zips: int = 10) -> int: + all_tags = gh.release_list_tags(repository) + + # Remove releases for deleted plugins (guarded on zips/ existing, as in Bash). + if os.path.isdir("zips"): + for release_dir in sorted(glob.glob("metadata/*/")): + plugin_name = os.path.basename(release_dir.rstrip("/")) + if not os.path.isdir(f"plugins/{plugin_name}"): + actions.log(f" Removing deleted plugin releases: {plugin_name}") + for tag in [t for t in all_tags if t.startswith(f"{plugin_name}-")]: + actions.log(f" Deleting release {tag}") + gh.release_delete(tag, repository) + shutil.rmtree(release_dir, ignore_errors=True) + + # Prune old versioned releases per plugin (keep the newest max_versioned_zips). + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + tags = _versioned_tags(all_tags, plugin_name) + if len(tags) <= max_versioned_zips: + continue + for old_tag in tags[max_versioned_zips:]: + actions.log(f" Removed release {old_tag} (over limit of {max_versioned_zips})") + gh.release_delete(old_tag, repository) + return 0 diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py new file mode 100644 index 00000000..5a7e8a16 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -0,0 +1,399 @@ +"""Manifest construction, byte-for-byte equivalent to generate-manifest.sh. + +The pure builders here assemble ``dict`` objects in the *exact* key order of the +jq programs in the Bash script and drop ``None`` the way +``with_entries(select(.value != null))`` does, so ``jsonio.dumps`` reproduces the +compact ``jq -c`` output. GPG signing and the "only re-sign when the payload +changed" logic are preserved in :func:`write_manifest_if_changed` / +:func:`sign_manifest`. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +from typing import Optional + +from ..core import jsonio +from ..core.jsonio import drop_none + + +def build_version_entry(metadata: dict, url: str, size, canonical_version: str) -> dict: + """A ``versions[]`` entry. + + With metadata present, all fields are populated (nulls dropped). With empty + metadata the Bash else-branch emits only ``{version,url,size}`` using the + tag-derived canonical version. + """ + if metadata: + return drop_none({ + "version": metadata.get("version"), + "commit_sha": metadata.get("commit_sha"), + "commit_sha_short": metadata.get("commit_sha_short"), + "build_timestamp": metadata.get("build_timestamp"), + "last_updated": metadata.get("last_updated"), + "checksum_md5": metadata.get("checksum_md5"), + "checksum_sha256": metadata.get("checksum_sha256"), + "min_dispatcharr_version": metadata.get("min_dispatcharr_version"), + "max_dispatcharr_version": metadata.get("max_dispatcharr_version"), + "source_url": metadata.get("source_url"), + "url": url, + "size": size, + }) + return {"version": canonical_version, "url": url, "size": size} + + +def override_current_min_max(versioned_zips: list[dict], current_version: str, + min_da: str, max_da: str) -> list[dict]: + """Overlay plugin.json min/max onto the current version's entry (metadata-only updates).""" + out = [] + for entry in versioned_zips: + if entry.get("version") == current_version: + merged = dict(entry) + merged["min_dispatcharr_version"] = min_da if min_da != "" else None + merged["max_dispatcharr_version"] = max_da if max_da != "" else None + out.append(drop_none(merged)) + else: + out.append(entry) + return out + + +def build_plugin_entry(plugin_raw: dict, plugin_name: str, latest_url: str, + registry_url: str, registry_name: str, + versioned_zips: list[dict], latest_metadata: dict, + latest_size_kb) -> dict: + """Per-plugin manifest ``.manifest`` payload (metadata//manifest.json).""" + def g(key): + return plugin_raw.get(key) + + latest = None + if latest_metadata: + latest = drop_none({ + "version": latest_metadata.get("version"), + "commit_sha": latest_metadata.get("commit_sha"), + "commit_sha_short": latest_metadata.get("commit_sha_short"), + "build_timestamp": latest_metadata.get("build_timestamp"), + "last_updated": latest_metadata.get("last_updated"), + "checksum_md5": latest_metadata.get("checksum_md5"), + "checksum_sha256": latest_metadata.get("checksum_sha256"), + "min_dispatcharr_version": g("min_dispatcharr_version"), + "max_dispatcharr_version": g("max_dispatcharr_version"), + "source_url": latest_metadata.get("source_url"), + "latest_url": latest_url, + "url": versioned_zips[0]["url"] if versioned_zips else None, + "size": latest_size_kb, + }) + + return drop_none({ + "slug": plugin_name, + "name": g("name"), + "description": g("description"), + "author": g("author"), + "maintainers": g("maintainers"), + "license": g("license"), + "deprecated": True if g("deprecated") is True else None, + "source_type": "external" if g("source_type") == "external" else None, + "source_url": g("source_url"), + "repo_url": g("repo_url"), + "discord_thread": g("discord_thread"), + "registry_url": registry_url, + "registry_name": registry_name, + "last_updated": latest_metadata.get("last_updated") if latest_metadata else None, + "latest": latest, + "versions": versioned_zips, + }) + + +def trim_description(desc: str) -> str: + """``desc[:197] + "..."`` when longer than 200 chars, else unchanged.""" + return (desc[:197] + "...") if len(desc) > 200 else desc + + +def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, + latest_size_kb, min_da: str, max_da: str, + latest_url: str, manifest_url: str) -> dict: + """Compact entry for the root manifest's ``plugins[]``.""" + license_id = plugin_raw.get("license") or "" + return drop_none({ + "slug": plugin_name, + "name": plugin_raw.get("name") or "", + "description": trim_description(plugin_raw.get("description") or ""), + "manifest_url": manifest_url, + "author": plugin_raw.get("author") or "", + "license": license_id if license_id != "" else None, + "deprecated": True if plugin_raw.get("deprecated") is True else None, + "last_updated": latest_metadata.get("last_updated") if latest_metadata else None, + "latest_version": latest_metadata.get("version") if latest_metadata else None, + "latest_md5": latest_metadata.get("checksum_md5") if latest_metadata else None, + "latest_sha256": latest_metadata.get("checksum_sha256") if latest_metadata else None, + "latest_url": latest_url, + "latest_size": latest_size_kb if _num(latest_size_kb) > 0 else None, + "min_dispatcharr_version": min_da if min_da != "" else None, + "max_dispatcharr_version": max_da if max_da != "" else None, + }) + + +def build_root_manifest(registry_url: str, registry_name: str, root_url: str, + root_entries: list[dict]) -> dict: + return { + "registry_url": registry_url, + "registry_name": registry_name, + "root_url": root_url, + "plugins": root_entries, + } + + +def _num(value) -> float: + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +# ---- on-disk wrapper + GPG signing ------------------------------------------ +def write_manifest_if_changed(dest: str, payload: dict, generated_at: str) -> bool: + """Write ``{generated_at, manifest}`` only if the compact payload changed. + + Returns True if written, False if skipped (unchanged), mirroring the Bash + return codes that gate re-signing. + """ + new_compact = jsonio.dumps(payload) + if os.path.isfile(dest): + try: + with open(dest, encoding="utf-8") as fh: + existing = json.load(fh) + if jsonio.dumps(existing.get("manifest")) == new_compact: + return False + except (OSError, ValueError): + pass + wrapper = {"generated_at": generated_at, "manifest": json.loads(new_compact)} + with open(dest, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(wrapper)) + return True + + +def sig_is_current(file: str, gpg_key_id: str) -> bool: + """True when the embedded ``.signature`` was issued by ``gpg_key_id``.""" + try: + with open(file, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + return False + sig = data.get("signature") + if not sig: + return False + with tempfile.NamedTemporaryFile("w", suffix=".asc", delete=False) as tmp: + tmp.write(sig + "\n") + tmp_path = tmp.name + try: + out = subprocess.run(["gpg", "--list-packets", tmp_path], + capture_output=True, text=True).stdout + finally: + os.unlink(tmp_path) + m = re.search(r"issuer key ID ([A-Fa-f0-9]{16})", out) + if not m: + return False + return m.group(1).upper() == (gpg_key_id or "").upper() + + +def sign_manifest(file: str, gpg_key_id: str, passphrase: Optional[str]) -> bool: + """Sign the ``.manifest`` payload and embed the armored signature. False on failure.""" + if not gpg_key_id: + return True + with open(file, encoding="utf-8") as fh: + data = json.load(fh) + payload = jsonio.dumps(data["manifest"]) + args = ["gpg", "--batch", "--yes", "--armor", "--detach-sign", + "--local-user", gpg_key_id, "--output", "-"] + if passphrase: + args += ["--passphrase", passphrase, "--pinentry-mode", "loopback"] + proc = subprocess.run(args, input=payload, capture_output=True, text=True) + sig = proc.stdout.rstrip("\n") # command-substitution parity: no trailing newline + if proc.returncode != 0 or not sig: + return False + data["signature"] = sig + with open(file, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(data)) + return True + + +# ---- GPG key setup + full generation orchestrator --------------------------- +def import_gpg_key(private_key: str) -> tuple[str, bool]: + """Import the armored key and return (key_id, signing_failed). + + Empty ``private_key`` -> ("", False) (signing disabled, not a failure). + """ + if not private_key: + from ..core import actions + actions.log("GPG_PRIVATE_KEY not set - signatures will be skipped.") + return "", False + subprocess.run(["gpg", "--batch", "--import"], input=private_key, + capture_output=True, text=True) + out = subprocess.run(["gpg", "--list-secret-keys", "--keyid-format", "LONG"], + capture_output=True, text=True).stdout + key_id = "" + for line in out.splitlines(): + if line.startswith("sec"): + parts = line.split() + if len(parts) >= 2 and "/" in parts[1]: + key_id = parts[1].split("/", 1)[1] + break + from ..core import actions + if key_id: + actions.log(f"GPG signing enabled (key: {key_id})") + return key_id, False + actions.warning("GPG key import succeeded but no usable secret key found - signatures will be skipped.") + return "", True + + +def _canonical(version: str) -> str: + return re.sub(r"-[0-9]+$", "", version) + + +def strip_signatures() -> None: + """Remove ``.signature`` from every manifest (no key configured or signing failed).""" + import glob as _glob + for f in _glob.glob("metadata/*/manifest.json") + ["manifest.json"]: + if not os.path.isfile(f): + continue + try: + with open(f, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + if "signature" in data: + del data["signature"] + with open(f, "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(data)) + + +def generate(source_branch: str, releases_branch: str, repository: str, + build_meta_dir: str) -> int: + """Full generate-manifest.sh: per-plugin manifests, root manifest, signing.""" + import datetime + import glob + from ..core import actions, gh + + generated_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + registry_url = f"https://github.com/{repository}" + registry_name = repository + root_url = f"https://github.com/{repository}/releases/download" + raw_releases_url = f"https://raw.githubusercontent.com/{repository}/{releases_branch}" + + key_id, signing_failed = import_gpg_key(os.environ.get("GPG_PRIVATE_KEY", "")) + passphrase = os.environ.get("GPG_PASSPHRASE") or None + + def maybe_sign(path: str) -> None: + nonlocal signing_failed + if not sign_manifest(path, key_id, passphrase): + actions.warning(f"GPG signing failed for {path} - all signatures will be removed.") + signing_failed = True + + def sign_if_needed(path: str, written: bool) -> None: + if written: + maybe_sign(path) + elif key_id and not signing_failed and not sig_is_current(path, key_id): + maybe_sign(path) + + all_tags = gh.release_list_tags(repository) + root_entries: list[dict] = [] + plugin_count = 0 + + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_file = os.path.join(plugin_dir, "plugin.json") + if not os.path.isfile(plugin_file): + continue + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_key = plugin_name.replace("-", "_") + with open(plugin_file, encoding="utf-8") as fh: + raw = json.load(fh) + current_version = str(raw.get("version")) + min_da = str(raw.get("min_dispatcharr_version") or "") + max_da = str(raw.get("max_dispatcharr_version") or "") + unlisted = raw.get("unlisted") is True + actions.log(f" {plugin_name}") + + existing_manifest_file = f"metadata/{plugin_name}/manifest.json" + os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + existing_manifest = None + if os.path.isfile(existing_manifest_file): + try: + with open(existing_manifest_file, encoding="utf-8") as fh: + existing_manifest = json.load(fh) + except ValueError: + existing_manifest = None + + from ..core.version import sort_versions_desc + prefix = f"{plugin_name}-" + raw_versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != f"{plugin_name}-latest"] + versioned_tags = [f"{prefix}{v}" for v in sort_versions_desc(raw_versions)] + + versioned_zips: list[dict] = [] + latest_metadata: dict = {} + latest_zip_version = "" + latest_size_kb = 0 + latest_size_set = False + + for release_tag in versioned_tags: + zip_version = release_tag[len(prefix):] + zip_url = f"{plugin_name}-{zip_version}/{plugin_name}-{zip_version}.zip" + canonical_version = _canonical(zip_version) + + metadata: dict = {} + fresh = os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{canonical_version}.json") + if build_meta_dir and os.path.isfile(fresh): + with open(fresh, encoding="utf-8") as fh: + metadata = json.load(fh) + elif existing_manifest is not None: + for entry in ((existing_manifest.get("manifest") or {}).get("versions") or []): + if entry.get("version") == canonical_version: + metadata = entry + break + + zip_size_kb = 0 + if metadata: + zip_size_kb = metadata.get("size_kb", metadata.get("size", 0)) or 0 + if not latest_size_set: + latest_size_kb = zip_size_kb + latest_size_set = True + + versioned_zips.append(build_version_entry(metadata, zip_url, zip_size_kb, canonical_version)) + if metadata and not latest_metadata: + latest_metadata = metadata + latest_zip_version = zip_version + + latest_url = "" + if latest_zip_version: + latest_url = f"{plugin_name}-{latest_zip_version}/{plugin_name}-{latest_zip_version}.zip" + + versioned_zips = override_current_min_max(versioned_zips, current_version, min_da, max_da) + + plugin_entry = build_plugin_entry(raw, plugin_name, latest_url, registry_url, + registry_name, versioned_zips, latest_metadata, + latest_size_kb) + written = write_manifest_if_changed(existing_manifest_file, plugin_entry, generated_at) + sign_if_needed(existing_manifest_file, written) + + if unlisted: + continue + + manifest_url = f"{raw_releases_url}/metadata/{plugin_name}/manifest.json" + root_entries.append(build_root_entry(raw, plugin_name, latest_metadata, + latest_size_kb, min_da, max_da, + latest_url, manifest_url)) + plugin_count += 1 + + inner_root = build_root_manifest(registry_url, registry_name, root_url, root_entries) + written = write_manifest_if_changed("manifest.json", inner_root, generated_at) + sign_if_needed("manifest.json", written) + + if signing_failed or not key_id: + actions.warning("Removing all manifest signatures (no GPG key configured or signing failed).") + strip_signatures() + + actions.log(f"Generated manifest.json with {plugin_count} plugin(s).") + return 0 diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py new file mode 100644 index 00000000..9885723b --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -0,0 +1,440 @@ +"""Per-plugin and root READMEs for the releases branch. + +Ports plugin-readmes.sh and releases-readme.sh. Output is kept identical to the +Bash except for one intentional correction, called out in the plan: the +deprecated-plugin "All Versions" count now comes from the per-plugin manifest's +``versions[]`` (same source the active section uses) instead of the removed +``zips//*.zip`` path, which always rendered 0. +""" + +from __future__ import annotations + +import datetime +import glob +import json +import os +from typing import Optional + +from ..core import git + + +def fmt_date(value: str) -> str: + """ISO8601 -> "Mon DD YYYY, HH:MM UTC"; returns the input unchanged on failure.""" + if not value: + return value + try: + text = value.replace("Z", "+00:00") + dt = datetime.datetime.fromisoformat(text) + if dt.tzinfo is not None: + dt = dt.astimezone(datetime.timezone.utc) + return dt.strftime("%b %d %Y, %H:%M UTC") + except ValueError: + return value + + +def shields_encode(s: str) -> str: + """shields.io path escaping: _ -> __, - -> --, space -> _ (order matters).""" + s = s.replace("_", "__") + s = s.replace("-", "--") + s = s.replace(" ", "_") + return s + + +def _load_json(path: str) -> Optional[dict]: + try: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + except (OSError, ValueError): + return None + + +def _g(data: Optional[dict], key: str, default: str = "") -> str: + if not data: + return default + val = data.get(key) + return default if val is None else str(val) + + +# --------------------------------------------------------------------------- # +# Per-plugin README (metadata//README.md) +# --------------------------------------------------------------------------- # +def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, + root_url: str, repository: str, source_branch: str, + last_updated: str, has_readme: bool, + source_readme: str = "") -> str: + name = _g(plugin_raw, "name") + description = _g(plugin_raw, "description") + author = _g(plugin_raw, "author") + maintainers = ", ".join(str(m) for m in (plugin_raw.get("maintainers") or [])) + repo_url = _g(plugin_raw, "repo_url") + discord_thread = _g(plugin_raw, "discord_thread") + license_id = _g(plugin_raw, "license") + min_da = _g(plugin_raw, "min_dispatcharr_version") + max_da = _g(plugin_raw, "max_dispatcharr_version") + version = _g(plugin_raw, "version") + + manifest_body = manifest.get("manifest", {}) if manifest else {} + latest = manifest_body.get("latest") or {} + latest_url_path = latest.get("latest_url") or "" + latest_full_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + + L: list[str] = [] + L.append("[Back to All Plugins](../../README.md)") + L.append("") + L.append(f"# {name}") + L.append("") + L.append(f"**Version:** `{version}` | **Author:** {author} | **Last Updated:** {fmt_date(last_updated)}") + L.append("") + L.append(description) + L.append("") + + badges = _badges(license_id, discord_thread, repo_url) + if badges: + L.append(badges) + L.append("") + compat = _compat_badges(min_da, max_da) + if compat: + L.append(compat) + L.append("") + + L.append("## Downloads") + L.append("") + L.append("### Latest Release") + L.append("") + if latest_full_url: + L.append(f"- **Download:** [`{plugin_name}-latest.zip`]({latest_full_url})") + build_ts = latest.get("build_timestamp") or "" + commit_sha = latest.get("commit_sha") or "" + commit_short = latest.get("commit_sha_short") or "" + md5 = latest.get("checksum_md5") or "" + sha256 = latest.get("checksum_sha256") or "" + if build_ts: + L.append(f"- **Built:** {fmt_date(build_ts)}") + if commit_sha: + L.append(f"- **Source Commit:** [`{commit_short}`](https://github.com/{repository}/commit/{commit_sha})") + if md5 or sha256: + L.append("") + L.append("**Checksums:**") + L.append("```") + if md5: + L.append(f"MD5: {md5}") + if sha256: + L.append(f"SHA256: {sha256}") + L.append("```") + + L.append("") + L.append("### All Versions") + L.append("") + L.append("| Version | Download | Built | Commit | MD5 | SHA256 |") + L.append("|---------|----------|-------|--------|-----|--------|") + for ver in (manifest_body.get("versions") or []): + v = ver.get("version") or "" + if not v: + continue + url_path = ver.get("url") or "" + full_url = f"{root_url}/{url_path}" if root_url and url_path else "" + commit_sha = ver.get("commit_sha") or "" + commit_short = ver.get("commit_sha_short") or "" + build_ts = ver.get("build_timestamp") or "" + md5 = ver.get("checksum_md5") or "" + sha256 = ver.get("checksum_sha256") or "" + build_date = fmt_date(build_ts) + commit_cell = f"[`{commit_short}`](https://github.com/{repository}/commit/{commit_sha})" if commit_sha else "-" + download_cell = f"[Download]({full_url})" if full_url else "-" + L.append(f"| `{v}` | {download_cell} | {build_date or '-'} | {commit_cell} | {md5 or '-'} | {sha256 or '-'} |") + + L.append("") + L.append("---") + L.append("") + footer = "" + if maintainers: + footer = f"**Maintainers:** {maintainers} | " + footer += (f"**Source:** [Browse Plugin]" + f"(https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name})") + L.append(footer) + L.append("") + L.append("**Metadata:** [View full manifest](./manifest.json)") + + if has_readme: + L.append("") + L.append("---") + L.append("") + L.append("## Plugin README") + L.append("") + L.append(source_readme.rstrip("\n")) + + return "\n".join(L) + "\n" + + +def _badges(license_id: str, discord_thread: str, repo_url: str) -> str: + badges = "" + if license_id: + badges = (f"[![License: {license_id}]" + f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" + f"(https://spdx.org/licenses/{license_id}.html)") + if discord_thread: + if badges: + badges += " " + badges += ("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" + f"({discord_thread})") + if repo_url: + if badges: + badges += " " + badges += ("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" + f"({repo_url})") + return badges + + +def _compat_badges(min_da: str, max_da: str) -> str: + if not (min_da or max_da): + return "" + out = "" + if min_da: + out = f"![Dispatcharr min](https://img.shields.io/badge/Dispatcharr_min-{shields_encode(min_da)}-brightgreen?style=flat-square)" + if max_da: + if out: + out += " " + out += f"![Dispatcharr max](https://img.shields.io/badge/Dispatcharr_max-{shields_encode(max_da)}-orange?style=flat-square)" + return out + + +def version_count_from_manifest(manifest_file: str) -> int: + """Deprecated-plugin fix: derive count from the manifest's versions[] (was ls zips/).""" + data = _load_json(manifest_file) + if not data: + return 0 + versions = (data.get("manifest") or {}).get("versions") + return len(versions) if isinstance(versions, list) else 0 + + +# --------------------------------------------------------------------------- # +# Root releases README (README.md on the releases branch) +# --------------------------------------------------------------------------- # +import re as _re + + +def anchor_for(name: str) -> str: + """GitHub-style heading anchor: lowercase, non [a-z0-9-] -> -, collapse runs.""" + a = name.lower() + a = _re.sub(r"[^a-z0-9-]", "-", a) + a = _re.sub(r"-+", "-", a) + return a + + +def table_row(plugin_raw: dict, deprecated_pass: bool) -> str: + name = _g(plugin_raw, "name") + version = _g(plugin_raw, "version") + author = _g(plugin_raw, "author") + description = _g(plugin_raw, "description") + license_cell = _g(plugin_raw, "license", "-") or "-" + suffix = " (deprecated)" if deprecated_pass else "" + return f"| [`{name}`](#{anchor_for(name)}){suffix} | `{version}` | {author} | {license_cell} | {description} |" + + +def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: dict, + manifest: Optional[dict], last_updated: str, commit_sha: str, + commit_sha_short: str, version_count, repository: str, + source_branch: str, releases_branch: str, root_url: str, + has_source_readme: bool) -> str: + name = _g(plugin_raw, "name") + version = _g(plugin_raw, "version") + author = _g(plugin_raw, "author") + description = _g(plugin_raw, "description") + maintainers = ", ".join(str(m) for m in (plugin_raw.get("maintainers") or [])) + license_id = _g(plugin_raw, "license") + min_da = _g(plugin_raw, "min_dispatcharr_version") + max_da = _g(plugin_raw, "max_dispatcharr_version") + repo_url = _g(plugin_raw, "repo_url") + discord_thread = _g(plugin_raw, "discord_thread") + + latest_url_path = "" + if manifest: + latest_url_path = ((manifest.get("manifest") or {}).get("latest") or {}).get("latest_url") or "" + zip_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + + source_url = f"https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name}" + readme_url = f"https://github.com/{repository}/blob/{source_branch}/plugins/{plugin_name}/README.md" + releases_readme_url = f"https://github.com/{repository}/blob/{releases_branch}/metadata/{plugin_name}/README.md" + commit_url = f"https://github.com/{repository}/commit/{commit_sha}" + releases_dir = f"./metadata/{plugin_name}" + + suffix = " (deprecated)" if is_deprecated else "" + + L: list[str] = [] + L.append(f"### [{name}]({releases_readme_url}){suffix}") + L.append("") + L.append(f"**Version:** `{version}` | **Author:** {author} | **Last Updated:** {fmt_date(last_updated)}") + L.append("") + L.append(description) + L.append("") + badges = _badges(license_id, discord_thread, repo_url) + if badges: + L.append(badges) + L.append("") + compat = _compat_badges(min_da, max_da) + if compat: + L.append(compat) + L.append("") + L.append("**Downloads:**") + if zip_url: + L.append(f"- [Latest Release (`{version}`)]({zip_url})") + L.append(f"- [All Versions ({version_count} available)]({releases_dir})") + L.append("") + footer = "" + if maintainers: + footer = f"**Maintainers:** {maintainers} | " + footer += f"**Source:** [Browse]({source_url})" + if has_source_readme: + footer += f" | [README]({readme_url})" + footer += f" | **Last Change:** [`{commit_sha_short}`]({commit_url})" + L.append(footer) + L.append("") + L.append("---") + L.append("") + return "\n".join(L) + "\n" + + +# --------------------------------------------------------------------------- # +# Orchestrators (IO): called from the releases-branch checkout directory +# --------------------------------------------------------------------------- # +def _now_iso() -> str: + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: + return git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now_iso() + + +def generate_plugin_readmes(source_branch: str, repository: str) -> None: + """Write metadata//README.md for every plugin (plugin-readmes.sh).""" + root = _load_json("manifest.json") or {} + root_url = (root.get("manifest") or {}).get("root_url") or "" + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_file = os.path.join(plugin_dir, "plugin.json") + plugin_raw = _load_json(plugin_file) + if plugin_raw is None: + continue + manifest_file = f"metadata/{plugin_name}/manifest.json" + manifest = _load_json(manifest_file) + if manifest is None: + continue + last_updated = _plugin_last_updated(source_branch, plugin_dir) + source_readme_path = os.path.join(plugin_dir, "README.md") + has_readme = os.path.isfile(source_readme_path) + source_readme = _read(source_readme_path) if has_readme else "" + content = render_plugin_readme( + plugin_name, plugin_raw, manifest, root_url, repository, source_branch, + last_updated, has_readme, source_readme) + with open(f"metadata/{plugin_name}/README.md", "w", encoding="utf-8") as fh: + fh.write(content) + + +def generate_releases_readme(source_branch: str, releases_branch: str, repository: str) -> None: + """Write the root README.md (releases-readme.sh).""" + root = _load_json("manifest.json") or {} + root_url = (root.get("manifest") or {}).get("root_url") or "" + + plugin_dirs = sorted(glob.glob("plugins/*/")) + raws: dict[str, dict] = {} + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = _load_json(os.path.join(pd, "plugin.json")) + if raw is not None: + raws[pname] = raw + + L: list[str] = [] + L.append("# Plugin Releases") + L.append("") + L.append("This branch contains all published plugin releases.") + L.append("") + L.append("## Quick Access") + L.append("") + L.append("- [manifest.json](./manifest.json) - Complete plugin registry with metadata") + L.append("- [metadata/](./metadata/) - Per-plugin manifests and READMEs") + L.append("") + L.append("## Available Plugins") + L.append("") + L.append("| Plugin | Version | Author | License | Description |") + L.append("|--------|---------|-------|---------|-------------|") + + for pass_name in ("active", "deprecated"): + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None: + continue + deprecated = raw.get("deprecated") is True + if raw.get("unlisted") is True: + continue + if pass_name == "active" and deprecated: + continue + if pass_name == "deprecated" and not deprecated: + continue + L.append(table_row(raw, pass_name == "deprecated")) + + L.append("") + L.append("---") + L.append("") + + def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: + pd = f"plugins/{pname}/" + manifest = _load_json(f"metadata/{pname}/manifest.json") + last_updated = _plugin_last_updated(source_branch, pd) + commit_sha = git.log_format("%H", f"origin/{source_branch}", pd) or "unknown" + commit_short = git.log_format("%h", f"origin/{source_branch}", pd) or "unknown" + version_count = version_count_from_manifest(f"metadata/{pname}/manifest.json") + has_source_readme = os.path.isfile(f"plugins/{pname}/README.md") + L.append(render_plugin_block( + is_deprecated=is_deprecated, plugin_name=pname, plugin_raw=raw, + manifest=manifest, last_updated=last_updated, commit_sha=commit_sha, + commit_sha_short=commit_short, version_count=version_count, + repository=repository, source_branch=source_branch, + releases_branch=releases_branch, root_url=root_url, + has_source_readme=has_source_readme)) + + # Active detailed sections + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None or raw.get("deprecated") is True or raw.get("unlisted") is True: + continue + emit_block(pname, raw, False) + + has_deprecated = any( + r.get("deprecated") is True for r in raws.values()) + if has_deprecated: + L.append("") + L.append("## Deprecated Plugins") + L.append("") + L.append("These plugins are deprecated and may be removed in the future.") + L.append("") + for pd in plugin_dirs: + pname = os.path.basename(pd.rstrip("/")) + raw = raws.get(pname) + if raw is None or raw.get("deprecated") is not True or raw.get("unlisted") is True: + continue + emit_block(pname, raw, True) + + L.append("## Using the Manifest") + L.append("") + L.append("Fetch `manifest.json` to programmatically access plugin metadata and download URLs:") + L.append("") + L.append("```bash") + L.append(f"curl https://raw.githubusercontent.com/{repository}/{releases_branch}/manifest.json") + L.append("```") + L.append("") + L.append("---") + L.append("") + L.append(f"*Last updated: {datetime.datetime.now(datetime.timezone.utc).strftime('%b %d %Y, %H:%M UTC')}*") + + with open("README.md", "w", encoding="utf-8") as fh: + fh.write("\n".join(L) + "\n") + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read() + except OSError: + return "" diff --git a/.github/pluginctl/src/pluginctl/publish/run.py b/.github/pluginctl/src/pluginctl/publish/run.py new file mode 100644 index 00000000..ea002d11 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/run.py @@ -0,0 +1,215 @@ +"""Publish orchestrator (port of run.sh). + +Clones the repo, sets up the releases branch, runs the build/cleanup/manifest/readme +phases, then commits and pushes, skipping commits whose only diff is timestamp +noise. Validated end to end in a fork per the rollout plan; the phase logic it +calls is unit-tested individually. +""" + +from __future__ import annotations + +import os +import subprocess +import tempfile + +from ..core import actions, git as _git_mod +from . import cleanup, manifest, readmes, zips + +RELEASES_BRANCH = "releases" +MAX_VERSIONED_ZIPS = 10 +RELEASES_BRANCH_VERSION = "3" + + +def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +def _configure_identity(repository: str, app_slug: str) -> None: + if app_slug: + from ..core import gh + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + _git("config", "user.name", f"{app_slug}[bot]") + if bot_user_id: + _git("config", "user.email", + f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com") + else: + _git("config", "user.email", f"{app_slug}[bot]@users.noreply.github.com") + else: + _git("config", "user.name", "github-actions[bot]") + _git("config", "user.email", + "41898282+github-actions[bot]@users.noreply.github.com") + + +def run(source_branch: str) -> int: + repository = os.environ["GITHUB_REPOSITORY"] + token = os.environ["GITHUB_TOKEN"] + app_slug = os.environ.get("APP_SLUG", "") + force_rebuild = os.environ.get("FORCE_REBUILD", "false") == "true" + force_plugin = os.environ.get("FORCE_REBUILD_PLUGIN", "") + remote = f"https://x-access-token:{token}@github.com/{repository}.git" + + actions.log(f"Publishing plugins from {source_branch} to {RELEASES_BRANCH}") + + workdir = tempfile.mkdtemp() + build_meta_dir = tempfile.mkdtemp() + start_cwd = os.getcwd() + try: + _git("clone", "--no-checkout", remote, f"{workdir}/repo") + os.chdir(f"{workdir}/repo") + _configure_identity(repository, app_slug) + _setup_releases_branch(repository, source_branch, token, remote, + force_rebuild, force_plugin) + + # Clean regenerated top-level artifacts and fetch source plugins. + for f in ("manifest.json", "README.md"): + if os.path.exists(f): + os.remove(f) + _git("fetch", "origin", source_branch) + _git("checkout", f"origin/{source_branch}", "--", "plugins") + os.makedirs("zips", exist_ok=True) + + if not force_rebuild: + current = _read("REPO_VER") + if current != RELEASES_BRANCH_VERSION: + actions.error("Releases branch version mismatch.") + actions.error(f" Expected : {RELEASES_BRANCH_VERSION}") + actions.error(f" Found : {current or '(none - migration not run)'}") + actions.error("Run the 'Migrate Releases to GitHub Releases' workflow first, then re-run.") + return 1 + + actions.log("\n=== Building ZIPs ===") + rc = zips.run(source_branch, repository, build_meta_dir) + if rc != 0: + return rc + actions.log("\n=== Cleaning up old releases ===") + cleanup.run(repository, MAX_VERSIONED_ZIPS) + actions.log("\n=== Generating manifests ===") + manifest.generate(source_branch, RELEASES_BRANCH, repository, build_meta_dir) + actions.log("\n=== Generating per-plugin READMEs ===") + readmes.generate_plugin_readmes(source_branch, repository) + actions.log("\n=== Generating releases README ===") + readmes.generate_releases_readme(source_branch, RELEASES_BRANCH, repository) + + return _commit_and_push(repository, source_branch, remote) + finally: + os.chdir(start_cwd) + subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) + + +def _setup_releases_branch(repository, source_branch, token, remote, + force_rebuild, force_plugin) -> None: + from ..core import gh + branch_exists = _git("ls-remote", "--exit-code", "--heads", "origin", + RELEASES_BRANCH, check=False).returncode == 0 + + if force_rebuild and force_plugin: + if branch_exists: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + else: + _orphan_init() + actions.log(f"Targeted force rebuild: deleting GitHub Releases for {force_plugin}") + for tag in gh.release_list_tags(repository): + if tag.startswith(f"{force_plugin}-"): + gh.release_delete(tag, repository) + mf = f"metadata/{force_plugin}/manifest.json" + if os.path.exists(mf): + os.remove(mf) + elif force_rebuild: + actions.log(f"Force rebuild requested - deleting all plugin GitHub Releases and resetting {RELEASES_BRANCH}.") + _git("fetch", "origin", source_branch, check=False) + _git("checkout", f"origin/{source_branch}", "--", "plugins", check=False) + if os.path.isdir("plugins"): + import glob + for plugin_dir in glob.glob("plugins/*/"): + pname = os.path.basename(plugin_dir.rstrip("/")) + for tag in gh.release_list_tags(repository): + if tag.startswith(f"{pname}-"): + gh.release_delete(tag, repository) + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("checkout", "--orphan", RELEASES_BRANCH) + _git("rm", "-rf", ".", check=False) + _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch (force rebuild)") + _git("push", "--force", remote, RELEASES_BRANCH) + elif branch_exists: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + else: + _orphan_init() + + +def _orphan_init() -> None: + _git("checkout", "--orphan", RELEASES_BRANCH) + _git("rm", "-rf", ".", check=False) + _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch") + + +def _commit_and_push(repository, source_branch, remote) -> int: + actions.log("\n=== Committing ===") + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("rm", "-rf", "--cached", "plugins", check=False) + with open("REPO_VER", "w", encoding="utf-8") as fh: + fh.write(RELEASES_BRANCH_VERSION + "\n") + _git("add", "metadata", "manifest.json", "README.md", "REPO_VER") + + if _git("diff", "--cached", "--quiet", check=False).returncode == 0: + actions.log("No changes to commit.") + return 0 + + if _only_timestamps(): + actions.log("No meaningful changes (only timestamps updated) - skipping commit.") + return 0 + + source_commit = _git("rev-parse", "--short", f"origin/{source_branch}").stdout.strip() + plugin_list = "" + changed = _read("changed_plugins.txt") + if changed.strip(): + plugin_list = "\n\n" + "\n".join(f"- {line}" for line in changed.splitlines() if line) + _git("commit", "-m", f"Publish plugin updates from {source_branch}\n\n" + f"Source commit: {source_commit}{plugin_list}") + _git("push", remote, RELEASES_BRANCH) + actions.log(f"Successfully published to {RELEASES_BRANCH}") + _emit_published(repository, changed) + return 0 + + +def _emit_published(repository: str, changed: str) -> None: + """Emit one plugin.published event per changed plugin (best-effort).""" + from ..integrations import webhooks + actor = os.environ.get("GITHUB_ACTOR", "") + for line in changed.splitlines(): + if "@" not in line: + continue + plugin_key, _, version = line.partition("@") + webhooks.emit("plugin.published", + webhooks.plugin_published(plugin_key.replace("_", "-"), version, None, actor)) + + +def _only_timestamps() -> bool: + """True when the staged diff is only README timestamp / manifest generated_at noise.""" + names = _git("diff", "--cached", "--name-only").stdout.splitlines() + for changed_file in names: + if changed_file == "README.md": + if _strip_lines(":README.md", "*Last updated:") != \ + _strip_lines("HEAD:README.md", "*Last updated:"): + return False + elif changed_file == "manifest.json": + if _strip_lines(":manifest.json", '"generated_at"') != \ + _strip_lines("HEAD:manifest.json", '"generated_at"'): + return False + else: + return False + return True + + +def _strip_lines(ref: str, needle: str) -> str: + out = _git("show", ref, check=False).stdout + return "\n".join(ln for ln in out.splitlines() if needle not in ln) + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + except OSError: + return "" diff --git a/.github/pluginctl/src/pluginctl/publish/yank.py b/.github/pluginctl/src/pluginctl/publish/yank.py new file mode 100644 index 00000000..fdc8963e --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/yank.py @@ -0,0 +1,264 @@ +"""Yank a plugin version from the releases branch (port of yank-version.sh). + +Removes a specific version's GitHub Release, regenerates manifests/READMEs, and, +when the yanked version was the latest (or only) one, opens a rollback PR against +the source branch so it is not republished. Authorized by an open issue whose +number is cited and (for non-latest yanks) closed. Env-driven, validated by fork +e2e per the rollout plan. +""" + +from __future__ import annotations + +import glob +import json +import os +import subprocess +import tempfile + +from ..core import actions, gh +from ..core.version import sort_versions_desc +from . import manifest, readmes + +RELEASES_BRANCH = "releases" + + +def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +def run() -> int: + repository = os.environ["GITHUB_REPOSITORY"] + token = os.environ["GITHUB_TOKEN"] + plugin = os.environ["YANK_PLUGIN"] + version = os.environ["YANK_VERSION"] + source_branch = os.environ["SOURCE_BRANCH"] + issue = os.environ["YANK_ISSUE"] + app_slug = os.environ.get("APP_SLUG", "") + remote = f"https://x-access-token:{token}@github.com/{repository}.git" + + actions.log(f"Yanking {plugin} v{version} from {RELEASES_BRANCH} (issue #{issue})") + + issue_state = gh.api(f"repos/{repository}/issues/{issue}", jq=".state") + if not issue_state: + actions.error(f"Issue #{issue} not found in {repository}.") + return 1 + if issue_state != "open": + actions.error(f"Issue #{issue} is already {issue_state}. Only open issues can authorize a yank.") + return 1 + actions.log(f" Issue #{issue} is open - proceeding.") + + workdir = tempfile.mkdtemp() + build_meta_dir = tempfile.mkdtemp() + os.environ["BUILD_META_DIR"] = build_meta_dir # kept for parity; generate() is passed it directly + start_cwd = os.getcwd() + try: + _git("clone", "--no-checkout", remote, f"{workdir}/repo") + os.chdir(f"{workdir}/repo") + _configure_identity(repository, app_slug) + + if _git("ls-remote", "--exit-code", "--heads", "origin", RELEASES_BRANCH, + check=False).returncode != 0: + actions.error("Releases branch does not exist.") + return 1 + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) + + release_tag = f"{plugin}-{version}" + plugin_manifest = f"metadata/{plugin}/manifest.json" + + if not gh.release_exists(release_tag, repository): + actions.error(f"GitHub Release {release_tag} not found. Nothing to yank.") + return 1 + + prefix = f"{plugin}-" + remaining = [t for t in gh.release_list_tags(repository) + if t.startswith(prefix) and t != f"{plugin}-latest" and t != release_tag] + is_last_version = len(remaining) == 0 + + current_latest = "" + if os.path.isfile(plugin_manifest): + try: + with open(plugin_manifest, encoding="utf-8") as fh: + data = json.load(fh) + current_latest = ((data.get("manifest") or {}).get("latest") or {}).get("version") or "" + except ValueError: + current_latest = "" + is_latest = version == current_latest + + actions.log(f" Current latest : {current_latest or 'unknown'}") + actions.log(f" Is latest : {str(is_latest).lower()}") + actions.log(f" Remaining releases: {len(remaining)}") + actions.log(f" Is last version : {str(is_last_version).lower()}") + + _git("fetch", "origin", source_branch) + _git("checkout", f"origin/{source_branch}", "--", "plugins") + source_type = "local" + pj = f"plugins/{plugin}/plugin.json" + if os.path.isfile(pj): + try: + with open(pj, encoding="utf-8") as fh: + source_type = (json.load(fh).get("source_type") or "local") + except ValueError: + source_type = "local" + + new_latest_version = "" + if is_last_version: + actions.log(f"Last version - deleting all GitHub Releases for {plugin}.") + gh.release_delete(release_tag, repository) + subprocess.run(["rm", "-rf", f"metadata/{plugin}"], check=False) + else: + actions.log(f"Deleting GitHub Release {release_tag}") + gh.release_delete(release_tag, repository) + if is_latest: + bare = [t[len(prefix):] for t in remaining] + ordered = sort_versions_desc(bare) + new_latest_version = ordered[0] if ordered else "" + if not new_latest_version: + actions.error("Could not find a replacement version to promote to latest.") + return 1 + actions.log(f"Promoting {new_latest_version} to latest (manifest will be updated by generate-manifest).") + + for f in ("manifest.json", "README.md"): + if os.path.exists(f): + os.remove(f) + actions.log("\n=== Regenerating manifests ===") + manifest.generate(source_branch, RELEASES_BRANCH, repository, build_meta_dir) + actions.log("\n=== Regenerating per-plugin READMEs ===") + readmes.generate_plugin_readmes(source_branch, repository) + actions.log("\n=== Regenerating releases README ===") + readmes.generate_releases_readme(source_branch, RELEASES_BRANCH, repository) + + releases_commit = _commit_releases(remote, plugin, version, issue) + rollback_pr_url = "" + if is_latest: + rollback_pr_url = _open_rollback_pr( + repository, source_branch, remote, plugin, version, + new_latest_version, is_last_version, source_type, issue) + + _finalize_issue(repository, plugin, version, issue, releases_commit, + is_latest, rollback_pr_url) + from ..integrations import webhooks + webhooks.emit("plugin.yanked", + webhooks.plugin_yanked(plugin, version, issue, rollback_pr_url or None)) + return 0 + finally: + os.chdir(start_cwd) + subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) + + +def _configure_identity(repository: str, app_slug: str) -> None: + if app_slug: + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + _git("config", "user.name", f"{app_slug}[bot]") + email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id + else f"{app_slug}[bot]@users.noreply.github.com") + _git("config", "user.email", email) + else: + _git("config", "user.name", "github-actions[bot]") + _git("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + + +def _commit_releases(remote, plugin, version, issue) -> str: + actions.log("\n=== Committing ===") + subprocess.run(["rm", "-rf", "plugins"], check=False) + _git("rm", "-rf", "--cached", "plugins", check=False) + _git("add", "metadata", "manifest.json", "README.md") + if _git("diff", "--cached", "--quiet", check=False).returncode == 0: + actions.log("No changes to commit - was this version already absent?") + return "" + _git("commit", "-m", f"Yank {plugin} v{version}\n\nRefs #{issue}") + releases_commit = _git("rev-parse", "--short", "HEAD").stdout.strip() + _git("push", remote, RELEASES_BRANCH) + actions.log(f"Successfully yanked {plugin} v{version} from {RELEASES_BRANCH}") + return releases_commit + + +def _open_rollback_pr(repository, source_branch, remote, plugin, version, + new_latest_version, is_last_version, source_type, issue) -> str: + actions.log("\n=== Opening rollback PR ===") + rollback_branch = f"yank/{plugin}-{version}" + _git("fetch", "origin", source_branch) + _git("checkout", "-b", rollback_branch, f"origin/{source_branch}") + + if is_last_version: + _git("rm", "-rf", f"plugins/{plugin}") + pr_title = f"[{plugin}]: Remove plugin (all versions yanked)" + pr_body = (f"## Plugin Removed\n\nAll published versions of `{plugin}` have been yanked " + f"from the releases branch.\n\nThis PR removes the plugin from the source branch " + f"to prevent it from being republished on the next run.\n\n" + f"**Yanked version:** `{version}`\n**Authorized by:** #{issue}\n\nCloses #{issue}") + else: + source_note = _rollback_source(plugin, new_latest_version, source_type) + _git("add", f"plugins/{plugin}/") + pr_title = f"[{plugin}]: Roll back to v{new_latest_version} (yank of v{version})" + pr_body = (f"## Version Rollback\n\n`{plugin}` `{version}` has been yanked from the releases " + f"branch. This PR rolls the source back to `{new_latest_version}` so the yanked " + f"version is not republished on the next run.\n\n" + f"**Yanked version:** `{version}`\n**New latest:** `{new_latest_version}`\n" + f"**Authorized by:** #{issue}\n\n{source_note}\n\nCloses #{issue}") + + _git("commit", "-m", pr_title) + _git("push", remote, rollback_branch) + gh.label_create("Rollback", "E11D48", "Version rollback opened by the yank workflow", repository) + url = gh.pr_create(repository, source_branch, rollback_branch, pr_title, pr_body, + labels=["Rollback"]) or "" + actions.log(f"Rollback PR opened: {rollback_branch} -> {source_branch} ({url})") + return url + + +def _rollback_source(plugin, new_latest_version, source_type) -> str: + pj = f"plugins/{plugin}/plugin.json" + if source_type == "local": + restore_commit = "" + shas = _git("log", "--all", "--format=%H", "--", pj).stdout.split() + for sha in shas: + show = _git("show", f"{sha}:{pj}", check=False) + if show.returncode != 0: + continue + try: + if json.loads(show.stdout).get("version") == new_latest_version: + restore_commit = sha + break + except ValueError: + continue + if restore_commit: + actions.log(f"Restoring plugins/{plugin}/ from commit {restore_commit}") + _git("checkout", restore_commit, "--", f"plugins/{plugin}/") + return (f"Plugin source files restored from commit `{restore_commit[:7]}` " + f"(the last commit where `{plugin}` was at `{new_latest_version}`).") + actions.warning(f"Could not find a commit for {plugin} v{new_latest_version} - " + "falling back to version field update only.") + _bump_version(pj, new_latest_version) + return ("⚠️ **Could not restore source files** - no commit found for " + f"`{new_latest_version}` (history may have been squashed). Only the `version` " + "field was updated. Please review the source files manually before merging.") + _bump_version(pj, new_latest_version) + return (f"Version field in `plugin.json` updated to `{new_latest_version}`. The ZIP will be " + "re-fetched from the external source URL on the next publish run.") + + +def _bump_version(pj: str, new_version: str) -> None: + with open(pj, encoding="utf-8") as fh: + data = json.load(fh) + data["version"] = new_version + # jq '.version = $v' rewrites pretty-printed with 2-space indent + trailing newline. + with open(pj, "w", encoding="utf-8") as fh: + json.dump(data, fh, indent=2, ensure_ascii=False) + fh.write("\n") + + +def _finalize_issue(repository, plugin, version, issue, releases_commit, + is_latest, rollback_pr_url) -> None: + commit_label = releases_commit or "unknown" + if is_latest: + gh.issue_comment(issue, repository, + f"`{plugin}` v`{version}` has been removed from the releases branch " + f"(commit `{commit_label}`).\n\nA rollback PR has been opened to update the " + f"source branch: {rollback_pr_url or '(see workflow run for link)'}\n\n" + "This issue will close automatically when that PR is merged.") + else: + gh.issue_comment(issue, repository, + f"`{plugin}` v`{version}` has been removed from the releases branch " + f"(commit `{commit_label}`).\n\nThis was not the latest version so no source " + "branch changes are needed.") + gh.issue_close(issue, repository, reason="completed") diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py new file mode 100644 index 00000000..584ba734 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -0,0 +1,145 @@ +"""Build versioned ZIPs + GitHub Releases (port of build-zips.sh). + +Runs from the releases-branch checkout directory. For every plugin whose current +version has no GitHub Release yet, it builds (or downloads, for external plugins) +the ZIP, records per-version metadata under ``BUILD_META_DIR`` for +generate-manifest to consume, uploads the release, and appends the plugin to +``changed_plugins.txt``. +""" + +from __future__ import annotations + +import glob +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import time +import urllib.request +from typing import Optional + +from ..core import actions, gh, git, jsonio +from ..core.jsonio import drop_none + + +def _now() -> str: + import datetime + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _checksums(path: str) -> tuple[str, str]: + md5 = hashlib.md5() + sha = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + md5.update(chunk) + sha.update(chunk) + return md5.hexdigest(), sha.hexdigest() + + +def _download(url: str, dest: str, attempts: int = 3) -> bool: + for attempt in range(1, attempts + 1): + try: + with urllib.request.urlopen(url, timeout=60) as resp, open(dest, "wb") as out: + shutil.copyfileobj(resp, out) + return True + except Exception: + if os.path.exists(dest): + os.remove(dest) + if attempt < attempts: + actions.log(f" Download attempt {attempt} failed, retrying in 15s...") + time.sleep(15) + return False + + +def run(source_branch: str, repository: str, build_meta_dir: str) -> int: + changed: list[str] = [] + open("changed_plugins.txt", "w", encoding="utf-8").close() + + for plugin_dir in sorted(glob.glob("plugins/*/")): + plugin_name = os.path.basename(plugin_dir.rstrip("/")) + plugin_key = plugin_name.replace("-", "_") + with open(os.path.join(plugin_dir, "plugin.json"), encoding="utf-8") as fh: + raw = json.load(fh) + version = str(raw.get("version")) + os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + + zip_path = f"/tmp/{plugin_name}-{version}.zip" + release_tag = f"{plugin_name}-{version}" + + if gh.release_exists(release_tag, repository): + actions.log(f" {plugin_name} v{version} - skipping (release already exists)") + continue + + source_type = raw.get("source_type") or "local" + build_timestamp = _now() + commit_sha = git.log_format("%H", f"origin/{source_branch}", plugin_dir) or "" + commit_sha_short = git.log_format("%h", f"origin/{source_branch}", plugin_dir) or "" + last_updated = git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now() + source_url_resolved = "" + + if source_type == "external": + source_url_resolved = str(raw.get("source_url") or "").replace("{version}", version) + actions.log(f" {plugin_name} v{version} - fetching external ZIP from {source_url_resolved}") + changed.append(f"{plugin_key}@{version}") + if not _download(source_url_resolved, zip_path): + actions.error(f"Failed to download external ZIP from {source_url_resolved} after 3 attempts") + return 1 + else: + actions.log(f" {plugin_name} v{version} - building") + changed.append(f"{plugin_key}@{version}") + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copytree(f"plugins/{plugin_name}", os.path.join(tmpdir, plugin_key)) + subprocess.run(["zip", "-r", zip_path, plugin_key], cwd=tmpdir, + check=True, stdout=subprocess.DEVNULL) + + checksum_md5, checksum_sha256 = _checksums(zip_path) + min_da = str(raw.get("min_dispatcharr_version") or "") + max_da = str(raw.get("max_dispatcharr_version") or "") + zip_size_kb = os.path.getsize(zip_path) // 1024 + + meta = drop_none({ + "version": version, + "commit_sha": commit_sha or None, + "commit_sha_short": commit_sha_short or None, + "build_timestamp": build_timestamp, + "last_updated": last_updated, + "checksum_md5": checksum_md5, + "checksum_sha256": checksum_sha256, + "min_dispatcharr_version": min_da or None, + "max_dispatcharr_version": max_da or None, + "source_url": source_url_resolved or None, + "size_kb": zip_size_kb, + }) + os.makedirs(os.path.join(build_meta_dir, plugin_key), exist_ok=True) + with open(os.path.join(build_meta_dir, plugin_key, f"{plugin_key}-{version}.json"), + "w", encoding="utf-8") as fh: + fh.write(jsonio.dumps(meta)) + + notes = _release_notes(repository, plugin_name, commit_sha, commit_sha_short) + actions.log(f" {plugin_name} v{version} - uploading to GitHub Releases") + gh.release_create(release_tag, repository, f"{plugin_name} v{version}", notes, zip_path) + os.remove(zip_path) + + with open("changed_plugins.txt", "w", encoding="utf-8") as fh: + for line in changed: + fh.write(line + "\n") + actions.log(f"Built {len(changed)} new/updated plugin(s).") + return 0 + + +def _release_notes(repository: str, plugin_name: str, commit_sha: str, + commit_sha_short: str) -> str: + readme_url = f"https://github.com/{repository}/blob/releases/metadata/{plugin_name}/README.md" + notes = "" + if commit_sha: + commit_url = f"https://github.com/{repository}/commit/{commit_sha}" + notes = f"**Commit:** [`{commit_sha_short}`]({commit_url})" + pr_number, pr_url = gh.commit_pull_number(repository, commit_sha) + if pr_number: + notes += f"\n**PR:** [#{pr_number}]({pr_url})" + notes += "\n" + notes += f"**README:** [Plugin README]({readme_url})" + return notes diff --git a/.github/pluginctl/src/pluginctl/validate/__init__.py b/.github/pluginctl/src/pluginctl/validate/__init__.py new file mode 100644 index 00000000..80fa5f5c --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/__init__.py @@ -0,0 +1 @@ +"""PR-time validation pipeline: detect, checks, SARIF, report, gate.""" diff --git a/.github/pluginctl/src/pluginctl/validate/clamav.py b/.github/pluginctl/src/pluginctl/validate/clamav.py new file mode 100644 index 00000000..880d439d --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/clamav.py @@ -0,0 +1,78 @@ +"""``pluginctl clamav-report``: parse clamscan output into status + findings table. + +Ports the "Set ClamAV status outputs" step of the clamav-scan job. ClamAV itself +still runs as the native CLI in the workflow; only the result parsing moves here. +""" + +from __future__ import annotations + +import hashlib +import os +from typing import Callable, Optional + +from ..core import actions + + +def _sha256_file(path: str) -> str: + try: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + except OSError: + return "" + + +def parse_infected_lines(output: str) -> list[str]: + """Lines ending in ' FOUND' (clamscan --infected format).""" + return [ln for ln in output.splitlines() if ln.endswith(" FOUND")] + + +def build_findings_table(infected_lines: list[str], workspace: str, + sha256_fn: Optional[Callable[[str], str]] = None) -> str: + """Render the ``| File | Signature |`` table, linking known hashes to VirusTotal.""" + sha256_fn = sha256_fn or _sha256_file + rows = ["| File | Signature |", "|------|-----------|"] + for line in infected_lines: + # ": FOUND" + full_path = line + idx = line.rfind(": ") + if idx != -1 and line.endswith(" FOUND"): + full_path = line[:idx] + sig = line[idx + 2:-len(" FOUND")] + else: + sig = "" + file_rel = full_path + prefix = (workspace or "") + "/" + if workspace and file_rel.startswith(prefix): + file_rel = file_rel[len(prefix):] + digest = sha256_fn(full_path) + if digest: + rows.append(f"| `{file_rel}` | [`{sig}`](https://www.virustotal.com/gui/file/{digest}) |") + else: + rows.append(f"| `{file_rel}` | `{sig}` |") + return "\n".join(rows) + "\n" + + +def run(output_file: str = "clamav-output.txt", scan_exit: int = 0, + findings_out: str = "clamav-findings.md") -> int: + workspace = os.environ.get("GITHUB_WORKSPACE", "") + output = "" + if os.path.isfile(output_file): + with open(output_file, encoding="utf-8") as fh: + output = fh.read() + infected_lines = parse_infected_lines(output) + actions.set_output("clamav_infected", str(len(infected_lines))) + + if infected_lines: + table = build_findings_table(infected_lines, workspace) + with open(findings_out, "w", encoding="utf-8") as fh: + fh.write(table) + actions.set_output("clamav_status", "failure") + return 0 + if scan_exit >= 2: + actions.set_output("clamav_status", "failure") + return 0 + actions.set_output("clamav_status", "success") + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py new file mode 100644 index 00000000..63c50c4d --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -0,0 +1,190 @@ +"""``pluginctl detect``: port of validate/detect-changes.sh + inline blacklist. + +Determines the changed-plugin matrix, whether the PR must be auto-closed, whether +it touches files outside ``plugins/`` without authorization, new/updated flags, +and the signing-key-changed warning. Emits the same ``$GITHUB_OUTPUT`` keys the +Bash script did so the downstream jobs are unchanged. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from typing import Optional + +from ..core import actions, gh, git + +SAFE_NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +PUB_KEY_PATH = ".github/scripts/keys/dispatcharr-plugins.pub" + + +def is_safe_name(name: str) -> bool: + return bool(SAFE_NAME_RE.match(name)) + + +def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) -> bool: + """True when pr_author is the base-branch author or in base maintainers. + + Reads from the base branch only, so a PR cannot self-grant permission. + """ + if not base_json_text: + return False + try: + data = json.loads(base_json_text) + except json.JSONDecodeError: + return False + author = data.get("author") or "" + maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] + return pr_author == author or pr_author in maintainers + + +@dataclass +class BlacklistResult: + matched: bool = False + reason: str = "" + + +def check_blacklists(pr_author: str, plugins: list[str], + author_blacklist: str, plugin_blacklist: str) -> BlacklistResult: + """Case-insensitive author/plugin blacklist check (whitespace-trimmed entries).""" + author_bl = (author_blacklist or "").strip() + plugin_bl = (plugin_blacklist or "").strip() + if not author_bl and not plugin_bl: + return BlacklistResult() + + if author_bl: + for entry in author_bl.split(","): + slug = entry.replace(" ", "") + if pr_author.lower() == slug.lower(): + actions.warning(f"PR author '{pr_author}' is on the author blacklist.") + return BlacklistResult(True, "author-blacklisted") + + if plugin_bl: + blocked = plugin_bl.split(",") + for plugin in plugins: + if not plugin: + continue + for entry in blocked: + slug = entry.replace(" ", "") + if plugin.lower() == slug.lower(): + actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") + return BlacklistResult(True, "plugin-blacklisted") + return BlacklistResult() + + +def run(pr_author: str, base_ref: str, head_ref: str = "", + author_blacklist: str = "", plugin_blacklist: str = "", + repo: str = "") -> int: + """Execute detection; write outputs; return process exit code (0 ok, 1 hard block).""" + owner, _, name = repo.partition("/") + + def write_access() -> bool: + return gh.has_write_access(owner, name, pr_author) + + # Bot-authored yank rollback PRs bypass plugin validation entirely. + if pr_author.endswith("[bot]") and head_ref.startswith("yank/"): + actions.log(f"Bot-authored yank rollback PR detected ({pr_author}, {head_ref}) " + "- skipping plugin validation.") + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="true", + outside_violation="false", pub_key_changed="false", + has_new_plugin="false", has_updated_plugin="false") + return 0 + + merge_base = git.merge_base(f"origin/{base_ref}", "HEAD") + changed = git.diff_name_only(merge_base, "HEAD") + outside_changes = [f for f in changed if not f.startswith("plugins/")] + has_outside_violation = bool(outside_changes) and not write_access() + + plugin_list = sorted({f.split("/")[1] for f in changed if f.startswith("plugins/") and len(f.split("/")) > 1}) + + if not plugin_list: + return _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) + + # Allowlist: only safe kebab-case names enter the matrix. + safe_list: list[str] = [] + for plugin in plugin_list: + if is_safe_name(plugin): + safe_list.append(plugin) + else: + actions.warning(f"Skipping plugin with unsafe folder name: '{plugin}'") + plugin_list = safe_list + + if not plugin_list: + actions.error("No valid plugin changes detected in this PR.") + actions.set_outputs(close_pr="true", close_reason="no-valid-plugins", + plugin_count="0", matrix="[]", + has_new_plugin="false", has_updated_plugin="false") + return 0 + + plugin_count = len(plugin_list) + + has_new_plugin = False + has_updated_plugin = False + for plugin in plugin_list: + if git.object_exists(f"origin/{base_ref}:plugins/{plugin}/plugin.json"): + has_updated_plugin = True + else: + has_new_plugin = True + + is_maintainer = write_access() + has_any_permission = is_maintainer + if not is_maintainer: + for plugin in plugin_list: + base_json = git.show(f"origin/{base_ref}:plugins/{plugin}/plugin.json") + if base_json and author_has_plugin_permission(pr_author, base_json): + has_any_permission = True + break + + close_pr = (not has_any_permission) and (not has_new_plugin) + + # Blacklist gate (only meaningful when not already closing). + close_reason = "unauthorized" if close_pr else "" + if not close_pr: + bl = check_blacklists(pr_author, plugin_list, author_blacklist, plugin_blacklist) + if bl.matched: + close_pr = True + close_reason = bl.reason + + matrix_json = json.dumps(plugin_list, separators=(",", ":")) + actions.set_output("matrix", matrix_json) + actions.set_output("plugin_count", str(plugin_count)) + actions.set_output("close_pr", "true" if close_pr else "false") + actions.set_output("skip_validation", "false") + actions.set_output("outside_violation", "true" if has_outside_violation else "false") + actions.set_output("close_reason", close_reason) + if outside_changes: + actions.set_output("outside_files", "\n".join(outside_changes)) + + pub_key_changed = is_maintainer and PUB_KEY_PATH in outside_changes + actions.set_output("pub_key_changed", "true" if pub_key_changed else "false") + actions.set_output("has_new_plugin", "true" if has_new_plugin else "false") + actions.set_output("has_updated_plugin", "true" if has_updated_plugin else "false") + + actions.log(f"Detected {plugin_count} plugin(s): {' '.join(plugin_list)}") + actions.log(f"close_pr={'true' if close_pr else 'false'}") + return 0 + + +def _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) -> int: + if has_outside_violation: + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="false", + outside_violation="true", + has_new_plugin="false", has_updated_plugin="false") + actions.set_output("outside_files", "\n".join(outside_changes)) + return 0 + if write_access(): + pub_key_changed = PUB_KEY_PATH in outside_changes + actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", + close_reason="", skip_validation="true", + outside_violation="false", + pub_key_changed="true" if pub_key_changed else "false", + has_new_plugin="false", has_updated_plugin="false") + if outside_changes: + actions.set_output("outside_files", "\n".join(outside_changes)) + actions.log("No plugin changes detected - skipping plugin validation (author has write access).") + return 0 + actions.error("No plugin changes detected in this PR.") + return 1 diff --git a/.github/pluginctl/src/pluginctl/validate/gate.py b/.github/pluginctl/src/pluginctl/validate/gate.py new file mode 100644 index 00000000..24506bd7 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/gate.py @@ -0,0 +1,43 @@ +"""``pluginctl gate``: port of the ``Plugin PR Check`` evaluation ladder. + +This is the single fixed status check referenced by branch protection. The +decision order and messages are preserved exactly; the quarantine-label check is +a separate step that runs first in the workflow and is modeled here too. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class GateResult: + ok: bool + message: str + + +def evaluate(detect_result: str, close_pr: str, skip_validation: str, + outside_violation: str, title_result: str, + codeql_result: str, codeql_status: str, + clamav_result: str, clamav_status: str, + validate_result: str, report_result: str) -> GateResult: + """Return pass/fail with the exact ::error:: message the Bash ladder emits.""" + if detect_result != "success": + return GateResult(False, "Plugin detection failed or no plugin changes found.") + if skip_validation == "true": + return GateResult(True, "No plugin changes detected and author has write access - passing.") + if title_result == "failure": + return GateResult(False, "PR title does not match the required format. Rename the PR and re-run.") + if outside_violation == "true": + return GateResult(False, "PR contains unauthorized changes outside the plugins/ directory.") + if close_pr == "true": + return GateResult(False, "PR is unauthorized - no permission to modify these plugins.") + if codeql_result == "failure" or codeql_status == "failure": + return GateResult(False, "CodeQL security analysis failed. See the Security tab for details.") + if clamav_result == "failure" or clamav_status == "failure": + return GateResult(False, "ClamAV antivirus scan detected threats. See PR comment for details.") + if validate_result in ("failure", "cancelled"): + return GateResult(False, "One or more plugin validations failed. See PR comment for details.") + if report_result != "success": + return GateResult(False, "Plugin validation failed. See PR comment for details.") + return GateResult(True, "All plugins validated successfully.") diff --git a/.github/pluginctl/src/pluginctl/validate/labels.py b/.github/pluginctl/src/pluginctl/validate/labels.py new file mode 100644 index 00000000..bf877b0e --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/labels.py @@ -0,0 +1,35 @@ +"""``pluginctl label``: port of the inline ``label-pr`` job. + +Reconciles the classification labels on a PR. The decision is pure and unit +tested; applying it just calls ``gh pr edit --add-label/--remove-label`` with the +same add-when-true / remove-when-false behavior as the Bash ``apply_label``. +""" + +from __future__ import annotations + +from ..core import gh + +LABELS = ["New Plugin", "Plugin Update", "Repo Update", "Invalid"] + + +def decide_labels(has_new_plugin: bool, has_updated_plugin: bool, + outside_files: str, outside_violation: bool, + close_pr: bool) -> dict[str, bool]: + """Return the desired on/off state for each managed label.""" + is_repo_update = bool(outside_files) and not outside_violation + is_invalid = outside_violation or close_pr + return { + "New Plugin": has_new_plugin, + "Plugin Update": has_updated_plugin, + "Repo Update": is_repo_update, + "Invalid": is_invalid, + } + + +def run(pr_number: str, has_new_plugin: bool, has_updated_plugin: bool, + outside_files: str, outside_violation: bool, close_pr: bool) -> int: + desired = decide_labels(has_new_plugin, has_updated_plugin, + outside_files, outside_violation, close_pr) + for label in LABELS: + gh.pr_edit_label(pr_number, label, desired[label]) + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/langs.py b/.github/pluginctl/src/pluginctl/validate/langs.py new file mode 100644 index 00000000..622eb4d7 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/langs.py @@ -0,0 +1,79 @@ +"""``pluginctl detect-langs``: which CodeQL languages are present in changed plugins. + +Ports the "Detect supported languages" bash step. Emits ``found`` / ``languages`` +/ ``unscanned_langs`` the same way, in the same order, so the CodeQL matrix and +the "unscanned file types" notice are unchanged. +""" + +from __future__ import annotations + +import os + +from ..core import actions + +# CodeQL-supported languages, in the Bash detection order. +_SUPPORTED = [ + ("python", {".py"}), + ("javascript", None), # special-cased (prunes vendored dirs) + ("go", {".go"}), + ("ruby", {".rb"}), + ("java-kotlin", {".java", ".kt", ".kts"}), + ("c-cpp", {".c", ".cpp", ".cc", ".h", ".hpp"}), +] + +# Present-but-unsupported file types, in the Bash order. +_UNSCANNED = [ + ("shell", {".sh", ".bash"}), + ("php", {".php"}), + ("lua", {".lua"}), + ("perl", {".pl", ".pm"}), + ("rust", {".rs"}), +] + +# Directories pruned from the JavaScript search (vendored / build output). +_JS_PRUNE = {"node_modules", "dist", "build", "static"} +_JS_EXTS = {".js", ".ts", ".jsx", ".tsx"} + + +def _has_ext(root: str, exts: set[str]) -> bool: + for _dirpath, _dirs, files in os.walk(root): + for fn in files: + if os.path.splitext(fn)[1] in exts: + return True + return False + + +def _has_js(root: str) -> bool: + for dirpath, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in _JS_PRUNE] + for fn in files: + if os.path.splitext(fn)[1] in _JS_EXTS: + return True + return False + + +def detect(root: str = "plugins") -> tuple[bool, str, str]: + """Return (found, languages_csv, unscanned_csv).""" + langs: list[str] = [] + for name, exts in _SUPPORTED: + if name == "javascript": + if _has_js(root): + langs.append(name) + elif _has_ext(root, exts): + langs.append(name) + unscanned = [name for name, exts in _UNSCANNED if _has_ext(root, exts)] + return bool(langs), ",".join(langs), ",".join(unscanned) + + +def run(root: str = "plugins") -> int: + found, languages, unscanned = detect(root) + actions.set_output("unscanned_langs", unscanned) + actions.set_output("found", "true" if found else "false") + actions.set_output("languages", languages) + if found: + actions.log(f"Detected languages for CodeQL: {languages}") + else: + actions.log("No supported language files found in changed plugins, skipping CodeQL.") + if unscanned: + actions.log(f"Unscanned file types (no CodeQL support): {unscanned}") + return 0 diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py new file mode 100644 index 00000000..965951da --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -0,0 +1,394 @@ +"""``pluginctl validate``: port of validate/validate.sh. + +Validates one plugin and writes a markdown report fragment identical to the Bash +output: same ``### Plugin:`` header, description/repo subtext, the +``| Check | Status | Details |`` table (rows hidden when they pass, in the same +order), the optional release/compare links, and the trailing ```` +tab-separated marker consumed by ``report``. Emits ``result``/``is_new``/ +``has_permission`` outputs and exits non-zero on failure. +""" + +from __future__ import annotations + +import json +import os +import re +import urllib.error +import urllib.request +from typing import Optional + +from ..core import actions, gh, git +from ..core.version import is_semver, is_dispatcharr_version, version_greater_than + +METADATA_ONLY_FIELDS = [ + "description", "repo_url", "discord_thread", "min_dispatcharr_version", + "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", +] + +_GH_DOWNLOAD_RE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/") +_GH_DOWNLOAD_TAG_RE = re.compile(r"^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/") + +SPDX_URL = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" + + +# ---- jq-parity scalar helpers ------------------------------------------------ +def jq_r(value, default: Optional[str] = None) -> str: + """Mimic ``jq -r``. Without a default, null/missing renders as ``"null"``. + + With ``default`` (the ``// "x"`` idiom), null/missing renders as ``default``. + """ + if value is None: + return "null" if default is None else default + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def field_present(raw: dict, key: str) -> bool: + """Reproduce ``jq -e '.\"key\"'``: present unless null or false.""" + val = raw.get(key, None) + return not (val is None or val is False) + + +def tsv(fields: list[str]) -> str: + """Reproduce ``@tsv`` escaping for a single row.""" + def esc(s: str) -> str: + return (s.replace("\\", "\\\\").replace("\t", "\\t") + .replace("\n", "\\n").replace("\r", "\\r")) + return "\t".join(esc(f) for f in fields) + + +# ---- network (injectable for tests) ----------------------------------------- +def http_status(url: str) -> str: + """Follow redirects, return the numeric HTTP status as a string ("000" on error).""" + try: + req = urllib.request.Request(url, method="GET") + with urllib.request.urlopen(req, timeout=15) as resp: + return str(getattr(resp, "status", None) or resp.getcode()) + except urllib.error.HTTPError as exc: # type: ignore[attr-defined] + return str(exc.code) + except Exception: + return "000" + + +def fetch_spdx() -> Optional[dict]: + try: + with urllib.request.urlopen(SPDX_URL, timeout=15) as resp: + return json.loads(resp.read().decode("utf-8")) + except Exception: + return None + + +# ---- helper: source file count (non-external) ------------------------------- +def count_source_files(plugin_dir: str) -> int: + ignore = {"plugin.json", "README.md", "logo.png"} + count = 0 + for root, _dirs, files in os.walk(plugin_dir): + for fn in files: + if fn not in ignore: + count += 1 + return count + + +def run(plugin_name: str, pr_author: str, base_ref: str, output_file: str, + repo: str = "") -> int: + owner, _, name = repo.partition("/") + plugin_dir = f"plugins/{plugin_name}" + plugin_json_path = f"{plugin_dir}/plugin.json" + + failed = False + is_new = False + has_permission = False + rows: list[str] = [] + pre_lines: list[str] = [] # header + desc/repo subtext + post_lines: list[str] = [] # release/compare link + meta_row: Optional[str] = None + + pre_lines.append(f"### Plugin: `{plugin_name}`") + pre_lines.append("") + + raw: Optional[dict] = None + if os.path.isfile(plugin_json_path): + try: + with open(plugin_json_path, encoding="utf-8") as fh: + raw = json.load(fh) + except (OSError, json.JSONDecodeError): + raw = None + if raw is not None: + desc = jq_r(raw.get("description"), "") + repo_url = jq_r(raw.get("repo_url"), "") + if desc: + pre_lines.append(f"_{desc}_") + pre_lines.append("") + if repo_url: + pre_lines.append(f"[Source Repository]({repo_url})") + pre_lines.append("") + + def emit_fragment() -> None: + lines = list(pre_lines) + lines.append("| Check | Status | Details |") + lines.append("|-------|:------:|---------|") + lines.extend(rows) + lines.append("") + lines.extend(post_lines) + if meta_row is not None: + lines.append(meta_row) + text = "\n".join(lines) + "\n" + _write(output_file, text) + + # Folder name format + if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", plugin_name): + rows.append(f"| Folder name | ❌ | Must be lowercase-kebab-case - got " + f"`{plugin_name}`, e.g. `my-plugin-name` |") + failed = True + + # plugin.json existence (early exit) + if not os.path.isfile(plugin_json_path): + rows.append("| `plugin.json` | ❌ | File missing |") + emit_fragment() + actions.set_outputs(result="fail", is_new="false", has_permission="false") + return 0 + + # JSON syntax (early exit) + if raw is None: + rows.append("| JSON syntax | ❌ | Invalid JSON in plugin.json |") + emit_fragment() + actions.set_outputs(result="fail", is_new="false", has_permission="false") + return 0 + + # Required fields + missing = [f"`{key}`" for key in ("name", "version", "description") + if not field_present(raw, key)] + if missing: + failed = True + rows.append(f"| Required fields | ❌ | Missing: {', '.join(missing)} |") + else: + rows.append("| Required fields | ✅ | All required fields present |") + + author = jq_r(raw.get("author"), "") + maintainers = [str(m) for m in (raw.get("maintainers") or []) if m is not None] + maintainers_sp = " ".join(maintainers) + version = jq_r(raw.get("version")) + + # Base branch plugin.json (for version bump + compare link) + base_json_text = git.show(f"origin/{base_ref}:{plugin_json_path}") + base_raw = None + old_version = "" + old_source_url_tmpl = "" + if base_json_text is not None: + try: + base_raw = json.loads(base_json_text) + old_version = jq_r(base_raw.get("version"), "") + old_source_url_tmpl = jq_r(base_raw.get("source_url"), "") + except json.JSONDecodeError: + base_raw = None + + # External source checks + source_type = jq_r(raw.get("source_type"), "local") + release_link = "" + compare_link = "" + gh_tag = "" + old_gh_tag = "" + if source_type != "external": + if count_source_files(plugin_dir) == 0: + rows.append(f"| Plugin files | ❌ | No source files found in " + f"`plugins/{plugin_name}/`. Add your plugin source (e.g. `main.py`) " + f"or set `\"source_type\": \"external\"` in `plugin.json` if your plugin " + f"is hosted elsewhere |") + failed = True + else: + ext_source_url = jq_r(raw.get("source_url"), "") + ext_repo_url = jq_r(raw.get("repo_url"), "") + if not ext_source_url: + rows.append("| `source_url` | ❌ | Required when `source_type` is `external` |") + failed = True + elif not ext_source_url.startswith("https://"): + rows.append("| `source_url` | ❌ | Must be an HTTPS URL |") + failed = True + elif "{version}" not in ext_source_url: + rows.append("| `source_url` | ❌ | Must contain a `{version}` placeholder " + "(e.g. `.../v{version}/plugin.zip`) |") + failed = True + else: + if is_semver(version): + resolved_url = ext_source_url.replace("{version}", version) + code = http_status(resolved_url) + if code == "200": + rows.append("| Release artifact | ✅ | Artifact reachable at resolved URL |") + m = _GH_DOWNLOAD_RE.match(resolved_url) + if m: + gh_owner, gh_repo, gh_tag = m.group(1), m.group(2), m.group(3) + release_link = f"https://github.com/{gh_owner}/{gh_repo}/releases/tag/{gh_tag}" + if old_version and old_source_url_tmpl: + old_resolved = old_source_url_tmpl.replace("{version}", old_version) + om = _GH_DOWNLOAD_TAG_RE.match(old_resolved) + if om: + old_gh_tag = om.group(1) + compare_link = (f"https://github.com/{gh_owner}/{gh_repo}" + f"/compare/{old_gh_tag}...{gh_tag}") + else: + rows.append(f"| Release artifact | ❌ | Could not reach `{resolved_url}` " + f"(HTTP `{code}`) — ensure the release exists |") + failed = True + if not ext_repo_url: + rows.append("| `repo_url` | ❌ | Required for external plugins — set to the upstream " + "source repository URL |") + failed = True + + # Maintainers + if not author and not maintainers_sp: + rows.append("| Maintainers | ❌ | At least one of `author` or `maintainers` must " + "include your GitHub username |") + failed = True + else: + parts = [] + if author: + parts.append(f"`{author}`") + for m in maintainers_sp.split(): + parts.append(f"`{m}`") + rows.append(f"| Maintainers | ✅ | {', '.join(parts)} |") + + # License + license_id = jq_r(raw.get("license"), "") + if not license_id: + rows.append("| License | ❌ | `license` is required - provide an " + "[OSI-approved SPDX identifier](https://spdx.org/licenses/) " + "(e.g. `MIT`, `Apache-2.0`) |") + failed = True + else: + spdx = fetch_spdx() + if spdx is None: + rows.append("| License | ⚠️ | Could not fetch SPDX license list - skipping validation |") + else: + osi = {l["licenseId"]: l.get("name", "") + for l in spdx.get("licenses", []) if l.get("isOsiApproved") is True} + if license_id in osi: + rows.append(f"| License | ✅ | `{license_id}` - {osi[license_id]} |") + else: + rows.append(f"| License | ❌ | `{license_id}` is not an " + "[OSI-approved SPDX identifier](https://spdx.org/licenses/) |") + failed = True + + # Permission + is_repo_maintainer = bool(owner) and gh.has_write_access(owner, name, pr_author) + if is_repo_maintainer: + rows.append("| Permission | ✅ | You have permission to modify this plugin |") + has_permission = True + elif base_raw is not None: + base_author = jq_r(base_raw.get("author"), "") + base_maintainers = [str(m) for m in (base_raw.get("maintainers") or []) if m is not None] + if pr_author == base_author or pr_author in base_maintainers: + rows.append("| Permission | ✅ | You have permission to modify this plugin |") + has_permission = True + else: + rows.append(f"| Permission | ❌ | `{pr_author}` is not listed in `author` or `maintainers` |") + failed = True + else: + if pr_author == author or pr_author in maintainers: + rows.append(f"| Permission | ✅ | New plugin - `{pr_author}` listed in `author`/`maintainers` |") + has_permission = True + else: + rows.append(f"| Permission | ❌ | Add `\"author\": \"{pr_author}\"` to plugin.json |") + failed = True + + # Version format + if is_semver(version): + rows.append(f"| Version | ✅ | `{version}` |") + else: + rows.append(f"| Version | ❌ | `{version}` is not valid semver - expected `X.Y.Z` |") + failed = True + + # Version bump + if base_raw is not None: + if version_greater_than(version, old_version): + rows.append(f"| Version bump | ✅ | `{old_version}` → `{version}` |") + else: + changed_fields = _changed_fields(base_raw, raw) + metadata_only = all(f in METADATA_ONLY_FIELDS for f in changed_fields) + if metadata_only and changed_fields: + rows.append(f"| Version bump | ✅ | `{old_version}` (unchanged - metadata-only update) |") + else: + other_changed = "" + if not changed_fields: + others = git.diff_name_only_range(f"origin/{base_ref}...HEAD", [plugin_dir]) + others = [f for f in others if f != plugin_json_path] + other_changed = others[0] if others else "" + if not changed_fields and not other_changed: + rows.append("| Version bump | ❌ | No changes detected - nothing to publish |") + else: + rows.append(f"| Version bump | ❌ | `{version}` must be greater than current `{old_version}` |") + failed = True + else: + rows.append("| Version bump | ✅ | New plugin |") + is_new = True + + # Dispatcharr version constraints + min_da = jq_r(raw.get("min_dispatcharr_version"), "") + max_da = jq_r(raw.get("max_dispatcharr_version"), "") + if min_da and not is_dispatcharr_version(min_da): + rows.append(f"| `min_dispatcharr_version` | ❌ | `{min_da}` is not valid semver - " + "expected `X.Y.Z` or `vX.Y.Z` |") + failed = True + if max_da and not is_dispatcharr_version(max_da): + rows.append(f"| `max_dispatcharr_version` | ❌ | `{max_da}` is not valid semver - " + "expected `X.Y.Z` or `vX.Y.Z` |") + failed = True + if min_da and max_da and is_dispatcharr_version(max_da) and is_dispatcharr_version(min_da): + _max = max_da[1:] if max_da.startswith("v") else max_da + _min = min_da[1:] if min_da.startswith("v") else min_da + if not version_greater_than(_max, _min) and _max != _min: + rows.append(f"| Version range | ❌ | `max_dispatcharr_version` (`{max_da}`) must be ≥ " + f"`min_dispatcharr_version` (`{min_da}`) |") + failed = True + + # Optional link fields + repo_url = jq_r(raw.get("repo_url"), "") + discord_thread = jq_r(raw.get("discord_thread"), "") + if repo_url and not re.match(r"^https?://", repo_url): + rows.append("| `repo_url` | ❌ | Must start with `http://` or `https://` |") + failed = True + if discord_thread and not re.match(r"^https?://", discord_thread): + rows.append("| `discord_thread` | ❌ | Must start with `http://` or `https://` |") + failed = True + + # Release / compare links (after the table) + if release_link: + link_line = f"[View release {gh_tag} on GitHub]({release_link})" + if compare_link: + link_line += f" · [Compare {old_gh_tag}...{gh_tag}]({compare_link})" + post_lines.append(link_line) + post_lines.append("") + + # Meta row + meta_fields = [ + jq_r(raw.get("name"), ""), + jq_r(raw.get("version"), ""), + jq_r(raw.get("description"), ""), + jq_r(raw.get("author"), ""), + ", ".join(maintainers), + jq_r(raw.get("repo_url"), ""), + jq_r(raw.get("discord_thread"), ""), + ] + meta_row = f"" + + emit_fragment() + actions.set_outputs( + result="pass" if not failed else "fail", + is_new="true" if is_new else "false", + has_permission="true" if has_permission else "false", + ) + return 1 if failed else 0 + + +def _changed_fields(old: dict, new: dict) -> list[str]: + """Keys of NEW whose value differs from OLD (missing old -> null), sorted like jq keys.""" + changed = [k for k in new.keys() if old.get(k) != new[k]] + return sorted(changed) + + +def _write(path: str, text: str) -> None: + if path in ("/dev/stdout", "-"): + print(text, end="") + return + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py new file mode 100644 index 00000000..4c430600 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -0,0 +1,425 @@ +"""``pluginctl report``: port of validate/report.sh. + +Combines per-plugin fragments, renders the single ```` +comment (same markers / sections / separators / emoji as the Bash), minimizes +prior validation comments as outdated, posts the new one, and closes the PR for +the unauthorized/blacklisted variants. The comment builder is pure so its exact +text is unit-testable; ``run`` performs the IO around it. +""" + +from __future__ import annotations + +import glob +import os +from dataclasses import dataclass, field +from typing import Optional + +from ..core import actions, gh, git + +MARKER = "" + + +@dataclass +class ParsedFragments: + combined_body: str = "" + plugin_links: str = "" + any_failed: bool = False + pr_plugin_names: list[str] = field(default_factory=list) + + +def _read(path: str) -> str: + try: + with open(path, encoding="utf-8") as fh: + return fh.read() + except OSError: + return "" + + +def parse_fragments(fragments_dir: str) -> ParsedFragments: + result = ParsedFragments() + for fragment in sorted(glob.glob(os.path.join(fragments_dir, "*.fragment.md"))): + content = _read(fragment) + if "❌" in content: + result.any_failed = True + # META_ROW extraction + for line in content.splitlines(): + if line.startswith(""): + meta = meta[:-3] + fields = meta.split("\t") + fields += [""] * (7 - len(fields)) + f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] + if f_repo or f_discord: + result.plugin_links += f"**`{f_name}`:**\n" + if f_repo: + result.plugin_links += f"- [GitHub Repository]({f_repo})\n" + if f_discord: + result.plugin_links += f"- [Discord Thread]({f_discord})\n" + result.plugin_links += "\n" + break + # visible = fragment minus META_ROW lines, trailing whitespace stripped ($() semantics) + visible_lines = [ln for ln in content.splitlines() if not ln.startswith("\n") + assert "# Plugin Validation Results" in c + assert "## 🎉 All validation checks passed!" in c + assert "This PR modifies **1** plugin(s) and all checks have passed." in c + + +def test_fragment_failure_shows_failed(): + c = report.build_comment(**_base_kwargs(fragment_failed=True)) + assert "## ❌ Validation failed" in c + assert "🎉" not in c + + +def test_unauthorized_close_variant(): + c = report.build_comment(**_base_kwargs(close_pr=True, close_reason="unauthorized")) + assert "## PR Closed: Unauthorized" in c + assert "automatically closed" in c + # main body / success block suppressed + assert "🎉" not in c + + +def test_blacklisted_author_variant(): + c = report.build_comment(**_base_kwargs(close_pr=True, close_reason="author-blacklisted")) + assert "## PR Closed: Account Restricted" in c + + +def test_codeql_high_block_and_separator(): + c = report.build_comment(**_base_kwargs( + codeql_result="failure", codeql_errors="2", + codeql_findings="| Rule | Location | Description |\n")) + assert "❌ **CodeQL found 2 high or critical issue(s)**" in c + assert "\n---\n" in c + assert "## ❌ Validation failed" in c + + +def test_clamav_failure_block(): + c = report.build_comment(**_base_kwargs( + clamav_result="failure", clamav_infected="3", + clamav_findings="| File | Signature |\n")) + assert "❌ **ClamAV detected 3 infected file(s)**." in c + + +def test_title_invalid_block(): + c = report.build_comment(**_base_kwargs( + title_valid="false", title_feedback="Bad prefix.", + title_suggestion="[repo]: x")) + assert "### ❌ PR Title Format" in c + assert "Bad prefix." in c + assert "**Suggested format:** `[repo]: x`" in c + assert "## ❌ Validation failed" in c + + +def test_codeql_skipped_notice_no_separator_when_only_success(): + c = report.build_comment(**_base_kwargs(codeql_result="success")) + # success + no unscanned -> no findings separator, still passes + assert "## 🎉 All validation checks passed!" in c + + +def test_parse_fragments(tmp_path): + f = tmp_path / "demo.fragment.md" + f.write_text("### Plugin: `demo`\n\n| ok |\n\n", + encoding="utf-8") + parsed = report.parse_fragments(str(tmp_path)) + assert parsed.pr_plugin_names == ["demo"] + assert "\n" + ) + assert frag.read_text(encoding="utf-8") == expected + + +def test_missing_plugin_json_early_exit(plugin_repo): + tmp_path, out = plugin_repo + os.remove(tmp_path / "plugins" / "demo-plugin" / "plugin.json") + frag = tmp_path / "frag.md" + rc = validate.run("demo-plugin", "alice", "main", str(frag), repo="org/repo") + assert rc == 0 + assert _outputs(out)["result"] == "fail" + text = frag.read_text(encoding="utf-8") + assert "| `plugin.json` | ❌ | File missing |" in text + assert "//' || true) - if [[ -n "$META_ROW" ]]; then - IFS=$'\t' read -r f_name f_version f_description f_author f_maintainers f_repo_url f_discord_thread <<< "$META_ROW" - TABLE_ROWS+="| $f_name | $f_version | $f_description | $f_author | $f_maintainers |"$'\n' - if [[ -n "$f_repo_url" || -n "$f_discord_thread" ]]; then - PLUGIN_LINKS+="**\`${f_name}\`:**"$'\n' - [[ -n "$f_repo_url" ]] && PLUGIN_LINKS+="- [GitHub Repository](${f_repo_url})"$'\n' - [[ -n "$f_discord_thread" ]] && PLUGIN_LINKS+="- [Discord Thread](${f_discord_thread})"$'\n' - PLUGIN_LINKS+=$'\n' - fi - fi - - # Strip internal marker lines from visible output - VISIBLE=$(grep -v '" - echo "" - echo "# Plugin Validation Results" - echo "" - echo "**Modified plugins:** $PLUGIN_COUNT" - echo "" - - if [[ "${CLOSE_REASON:-}" == "no-valid-plugins" ]]; then - # echo "" - # echo "## Invalid Plugin Folder Name" - echo "" - echo "⚠️ Your PR modifies plugin folder(s) whose names do not meet the naming requirements. Plugin folder names must be **lowercase letters, numbers, and hyphens only** (e.g. \`my-plugin\`). Spaces and other special characters are not allowed." - echo "" - echo "Please rename the folder(s) and update your PR." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For help: [Dispatcharr Discord]($DISCORD_URL)" - fi - elif [[ "${CLOSE_REASON:-}" == "author-blacklisted" ]]; then - echo "" - echo "## PR Closed: Account Restricted" - echo "" - echo "Your GitHub account (\`$PR_AUTHOR\`) is not permitted to submit plugins to this repository. This PR has been automatically closed." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "If you believe this is an error, please reach out via the [Dispatcharr Discord]($DISCORD_URL)." - fi - elif [[ "${CLOSE_REASON:-}" == "plugin-blacklisted" ]]; then - echo "" - echo "## PR Closed: Plugin Restricted" - echo "" - echo "One or more plugins in this PR are on the restricted list and cannot be submitted to this repository. This PR has been automatically closed." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "If you believe this is an error, please reach out via the [Dispatcharr Discord]($DISCORD_URL)." - fi - elif [[ "$CLOSE_PR" == "true" ]]; then - echo "" - echo "## PR Closed: Unauthorized" - echo "" - echo "Your GitHub username (\`$PR_AUTHOR\`) does not appear in \`author\` or \`maintainers\` for any of the plugin(s) in this PR. This PR has been automatically closed." - echo "If you would like to contribute to this plugin, please consider reaching out to the maintainers of this plugin on Discord, or the plugin's Github repository." - echo "" - echo "If you are submitting a new plugin, add your GitHub username to the \`author\` field in your \`plugin.json\`." - if [[ -n "$PLUGIN_LINKS" ]]; then - echo "" - echo "### Plugin Contact Links" - echo "" - echo "$PLUGIN_LINKS" - fi - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For general help or plugin discussion:" - echo "- [Dispatcharr Discord]($DISCORD_URL)" - fi - else - echo "$COMBINED_BODY" - - if [[ -n "${OUTSIDE_FILES:-}" && "${OUTSIDE_VIOLATION:-}" == "true" ]]; then - OVERALL_FAILED=1 - echo "" - echo "⚠️ This PR modifies files outside of \`plugins/\`, which requires write access to the repository. These changes will block merging." - echo "" - echo "External contributions to repository tooling and scripts are not accepted via PR. If you think something needs fixing, please [open an issue](https://github.com/${GITHUB_REPOSITORY}/issues/new/choose) instead." - echo "" - echo "**Modified files:**" - echo "\`\`\`" - echo "${OUTSIDE_FILES}" - echo "\`\`\`" - echo "" - echo "Remove these changes and resubmit with only modifications inside \`plugins/\`." - if [[ -n "${DISCORD_URL:-}" ]]; then - echo "" - echo "For help: [Dispatcharr Discord]($DISCORD_URL)" - fi - echo "" - fi - - if [[ "${PUB_KEY_CHANGED:-}" == "true" ]]; then - echo "" - echo "---" - echo "" - echo "### ⚠️ Signing Key Change Detected" - echo "" - echo "This PR modifies \`.github/scripts/keys/dispatcharr-plugins.pub\`. This is the public GPG key used by Dispatcharr to verify manifest signatures." - echo "" - echo "**Before merging, confirm:**" - echo "- The corresponding private key and passphrase secrets (\`GPG_PRIVATE_KEY\`, \`GPG_PASSPHRASE\`) have been updated in the repository settings." - echo "- The new public key has been bundled into the Dispatcharr application." - echo "- Existing embedded signatures in \`manifest.json\` files on the \`releases\` branch will be regenerated on next publish." - echo "" - fi - - # insert --- if there are ANY codeql/clamav findings, medium, low, or skip/unscanned notice - if [[ -n "${CODEQL_RESULT:-}" && "${CODEQL_RESULT:-}" != "skipped" && "${CODEQL_RESULT:-}" != "success" ]] || \ - [[ -n "${CODEQL_MEDIUMS:-}" && "${CODEQL_MEDIUMS}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]] || \ - [[ -n "${CODEQL_LOWS:-}" && "${CODEQL_LOWS}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]] || \ - [[ -n "${CODEQL_SUPPRESSED:-}" && "${CODEQL_SUPPRESSED}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]] || \ - [[ "${CODEQL_RESULT:-}" == "skipped" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]] || \ - [[ "${CODEQL_RESULT:-}" != "skipped" && -n "${CODEQL_RESULT:-}" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]] || \ - [[ "${CLAMAV_RESULT:-}" == "failure" ]]; then - echo "" - echo "---" - echo "" - fi - - if [[ "${CLAMAV_RESULT:-}" == "failure" ]]; then - OVERALL_FAILED=1 - INFECTED_LABEL="${CLAMAV_INFECTED:-unknown}" - echo "" - echo "❌ **ClamAV detected $INFECTED_LABEL infected file(s)**." - echo "" - if [[ -f "clamav-findings/clamav-findings.md" ]]; then - cat "clamav-findings/clamav-findings.md" - fi - fi - - if [[ -n "${CODEQL_RESULT:-}" && "${CODEQL_RESULT:-}" != "skipped" && "${CODEQL_RESULT:-}" != "success" ]]; then - # echo "" - # echo "## Code Quality" - echo "" - OVERALL_FAILED=1 - ERROR_LABEL="${CODEQL_ERRORS:-unknown}" - echo "❌ **CodeQL found $ERROR_LABEL high or critical issue(s)** - these must be fixed before merging." - echo "" - if [[ -f "codeql-findings/codeql-findings.md" ]]; then - cat "codeql-findings/codeql-findings.md" - fi - fi - - if [[ -n "${CODEQL_MEDIUMS:-}" && "${CODEQL_MEDIUMS:-}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]]; then - echo "" - echo "**CodeQL found ${CODEQL_MEDIUMS} medium severity issue(s)**" - echo "These are not blocking, but are included for visibility." - echo "" - if [[ -f "codeql-medium-findings/codeql-medium-findings.md" ]]; then - cat "codeql-medium-findings/codeql-medium-findings.md" - fi - fi - - if [[ -n "${CODEQL_LOWS:-}" && "${CODEQL_LOWS:-}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]]; then - echo "" - echo "
" - echo "CodeQL found ${CODEQL_LOWS} low severity or informational result(s)" - echo "These are not blocking, but are included for visibility." - echo "" - if [[ -f "codeql-low-findings/codeql-low-findings.md" ]]; then - cat "codeql-low-findings/codeql-low-findings.md" - fi - echo "" - echo "
" - fi - - if [[ -n "${CODEQL_SUPPRESSED:-}" && "${CODEQL_SUPPRESSED:-}" != "0" && "${CODEQL_RESULT:-}" != "skipped" ]]; then - echo "" - echo "**${CODEQL_SUPPRESSED} finding(s) suppressed via inline \`codeql[...]\` comment** - requires maintainer review before merging; auto-merge is blocked for this PR." - echo "" - if [[ -f "codeql-suppressed-findings/codeql-suppressed-findings.md" ]]; then - cat "codeql-suppressed-findings/codeql-suppressed-findings.md" - fi - fi - - # CodeQL skipped notice (when no scannable files exist but unscannable types were found) - if [[ "${CODEQL_RESULT:-}" == "skipped" && -n "${CODEQL_UNSCANNED_LANGS:-}" ]]; then - UNSCANNED_DISPLAY=$(echo "${CODEQL_UNSCANNED_LANGS}" | tr ',' ' ') - echo "" - echo "**CodeQL analysis was skipped** - no supported source files were found. The following bundled file type(s) are not covered by CodeQL: \`${UNSCANNED_DISPLAY}\`." - echo "" - elif [[ -n "${CODEQL_UNSCANNED_LANGS:-}" && "${CODEQL_RESULT:-}" != "skipped" && -n "${CODEQL_RESULT:-}" ]]; then - UNSCANNED_DISPLAY=$(echo "${CODEQL_UNSCANNED_LANGS}" | tr ',' ' ') - echo "" - echo "**Note:** The following bundled file type(s) were not scanned by CodeQL (unsupported language): \`${UNSCANNED_DISPLAY}\`." - echo "" - fi - - if [[ -n "${TITLE_VALID:-}" && "${TITLE_VALID}" != "true" ]]; then - echo "" - echo "---" - echo "" - echo "### ❌ PR Title Format" - echo "" - echo "${TITLE_FEEDBACK}" - if [[ -n "${TITLE_SUGGESTION:-}" ]]; then - echo "" - echo "**Suggested format:** \`${TITLE_SUGGESTION}\`" - fi - echo "" - fi - - echo "" - echo "---" - echo "" - if [[ $OVERALL_FAILED -eq 0 ]]; then - echo "## 🎉 All validation checks passed!" - echo "" - echo "This PR modifies **$PLUGIN_COUNT** plugin(s) and all checks have passed." - else - echo "## ❌ Validation failed" - echo "" - echo "Some checks failed. Please review the errors above and update your PR." - fi - - if [[ -n "$OTHER_PLUGINS_SECTION" ]]; then - echo "" - echo "---" - echo "" - echo "$OTHER_PLUGINS_SECTION" - fi - - # if [[ -n "$PLUGIN_LINKS" ]]; then - # echo "" - # echo "---" - # echo "" - # echo "## Plugin Links" - # echo "" - # echo "$PLUGIN_LINKS" - # fi - - # if [[ -n "$TABLE_ROWS" ]]; then - # echo "" - # echo "---" - # echo "" - # echo "## Plugin Metadata" - # echo "" - # echo "$TABLE_HEADER" - # echo "$TABLE_SEP" - # echo "$TABLE_ROWS" - # fi - fi -} > pr_comment.txt - -# Minimize all previous validation comments as outdated before posting the new one -OWNER="${GITHUB_REPOSITORY%%/*}" -REPO="${GITHUB_REPOSITORY##*/}" - -PREV_NODE_IDS=$(gh api graphql -f query=' - query($owner: String!, $repo: String!, $number: Int!) { - repository(owner: $owner, name: $repo) { - pullRequest(number: $number) { - comments(first: 100) { - nodes { id body } - } - } - } - } -' -f owner="$OWNER" -f repo="$REPO" -F number="$PR_NUMBER" \ - --jq '.data.repository.pullRequest.comments.nodes[] - | select(.body | contains("")) - | .id' 2>/dev/null || true) - -if [[ -n "$PREV_NODE_IDS" ]]; then - while IFS= read -r node_id; do - gh api graphql -f query=' - mutation($id: ID!) { - minimizeComment(input: {subjectId: $id, classifier: OUTDATED}) { - minimizedComment { isMinimized } - } - } - ' -f id="$node_id" > /dev/null 2>&1 || true - done <<< "$PREV_NODE_IDS" - echo "Minimized $(echo "$PREV_NODE_IDS" | wc -l | tr -d ' ') previous validation comment(s) as outdated" -fi - -# Post PR comment - script succeeds/fails based on whether the comment posted -gh pr comment "$PR_NUMBER" --body "$(cat pr_comment.txt)" -COMMENT_EXIT=$? - -# Close PR for unauthorized plugin modifications -if [[ "$CLOSE_PR" == "true" && ( "${CLOSE_REASON:-}" == "unauthorized" || "${CLOSE_REASON:-}" == "author-blacklisted" || "${CLOSE_REASON:-}" == "plugin-blacklisted" ) ]]; then - gh pr close "$PR_NUMBER" - echo "PR #$PR_NUMBER closed: unauthorized" - exit $COMMENT_EXIT -fi - -exit $COMMENT_EXIT diff --git a/.github/scripts/validate/validate.sh b/.github/scripts/validate/validate.sh deleted file mode 100755 index 2dae7d48..00000000 --- a/.github/scripts/validate/validate.sh +++ /dev/null @@ -1,425 +0,0 @@ -#!/bin/bash -set -e - -# validate-single-plugin.sh -# Validates one plugin and writes a markdown report fragment to a file. -# -# Usage: validate-single-plugin.sh -# -# Arguments: -# plugin_name - Plugin folder name (e.g. my-plugin) -# pr_author - GitHub username of PR author -# base_ref - Base branch reference (e.g. main) -# output_file - File path to write the markdown report fragment to -# -# Outputs (written to $GITHUB_OUTPUT): -# result - "pass" or "fail" -# is_new - "true" if this is a new plugin (not on base branch) -# has_permission - "true" if pr_author is permitted to modify this plugin -# -# Environment variables required: -# GITHUB_REPOSITORY - Full repository name (owner/repo) -# GH_TOKEN - GitHub token for API access - -PLUGIN_NAME=$1 -PR_AUTHOR=$2 -BASE_REF=$3 -OUTPUT_FILE=${4:-/dev/stdout} - -if [[ -z "$PLUGIN_NAME" || -z "$PR_AUTHOR" || -z "$BASE_REF" ]]; then - echo "Usage: $0 [output_file]" - exit 1 -fi - -REPO_OWNER=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f1) -REPO_NAME=$(echo "$GITHUB_REPOSITORY" | cut -d'/' -f2) - -PLUGIN_DIR="plugins/$PLUGIN_NAME" -PLUGIN_JSON="$PLUGIN_DIR/plugin.json" -README="$PLUGIN_DIR/README.md" - -check_repo_maintainer() { - local author=$1 - PERMISSION=$(gh api repos/$REPO_OWNER/$REPO_NAME/collaborators/$author/permission --jq .permission 2>/dev/null || echo "none") - if [[ "$PERMISSION" == "admin" || "$PERMISSION" == "maintain" || "$PERMISSION" == "write" ]]; then - echo "1" - else - echo "0" - fi -} - -validate_semver() { - local version=$1 - if [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "1"; else echo "0"; fi -} - -validate_dispatcharr_version() { - local version=$1 - if [[ "$version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "1"; else echo "0"; fi -} - -version_greater_than() { - local new_version=$1 - local old_version=$2 - IFS='.' read -r NEW_MAJOR NEW_MINOR NEW_PATCH <<< "$new_version" - IFS='.' read -r OLD_MAJOR OLD_MINOR OLD_PATCH <<< "$old_version" - if (( 10#$NEW_MAJOR > 10#$OLD_MAJOR )); then return 0; fi - if (( 10#$NEW_MAJOR < 10#$OLD_MAJOR )); then return 1; fi - if (( 10#$NEW_MINOR > 10#$OLD_MINOR )); then return 0; fi - if (( 10#$NEW_MINOR < 10#$OLD_MINOR )); then return 1; fi - if (( 10#$NEW_PATCH > 10#$OLD_PATCH )); then return 0; fi - return 1 -} - -failed=0 -is_new="false" -has_permission="false" - -{ - echo "### Plugin: \`$PLUGIN_NAME\`" - echo "" - # Show description and repo link as header subtext if plugin.json is readable - if [[ -f "$PLUGIN_JSON" ]]; then - _desc=$(jq -r '.description // ""' "$PLUGIN_JSON" 2>/dev/null || true) - REPO_URL=$(jq -r '.repo_url // ""' "$PLUGIN_JSON" 2>/dev/null || true) - if [[ -n "$_desc" ]]; then - echo "_${_desc}_" - echo "" - fi - if [[ -n "$REPO_URL" ]]; then - echo "[Source Repository]($REPO_URL)" - echo "" - fi - fi - - TABLE_ROWS=() - - print_table() { - echo "| Check | Status | Details |" - echo "|-------|:------:|---------|" - for row in "${TABLE_ROWS[@]}"; do - echo "$row" - done - echo "" - } - - # ── Structural checks (hidden if pass) ─────────────────────────────────────── - - # Folder name format - if [[ ! "$PLUGIN_NAME" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then - TABLE_ROWS+=("| Folder name | ❌ | Must be lowercase-kebab-case - got \`$PLUGIN_NAME\`, e.g. \`my-plugin-name\` |") - failed=1 - fi - - # plugin.json existence (early exit) - if [[ ! -f "$PLUGIN_JSON" ]]; then - TABLE_ROWS+=("| \`plugin.json\` | ❌ | File missing |") - print_table - echo "result=fail" >> "$GITHUB_OUTPUT" - echo "is_new=false" >> "$GITHUB_OUTPUT" - echo "has_permission=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # JSON syntax (early exit) - if ! jq empty "$PLUGIN_JSON" 2>/dev/null; then - TABLE_ROWS+=("| JSON syntax | ❌ | Invalid JSON in plugin.json |") - print_table - echo "result=fail" >> "$GITHUB_OUTPUT" - echo "is_new=false" >> "$GITHUB_OUTPUT" - echo "has_permission=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # ── Required content ────────────────────────────────────────────────────────── - - # Required fields: name, version, description - shown as a combined row - MISSING_FIELDS=() - for key in name version description; do - if ! jq -e ".\"$key\"" "$PLUGIN_JSON" >/dev/null 2>&1; then - MISSING_FIELDS+=("\`$key\`") - failed=1 - fi - done - if [[ ${#MISSING_FIELDS[@]} -gt 0 ]]; then - MISSING_LIST=$(IFS=", "; echo "${MISSING_FIELDS[*]}") - TABLE_ROWS+=("| Required fields | ❌ | Missing: $MISSING_LIST |") - else - TABLE_ROWS+=("| Required fields | ✅ | All required fields present |") - fi - - # Extract metadata - AUTHOR=$(jq -r '.author // ""' "$PLUGIN_JSON") - MAINTAINERS=$(jq -r '[.maintainers[]?] | join(" ")' "$PLUGIN_JSON") - VERSION=$(jq -r '.version' "$PLUGIN_JSON") - - # Pre-fetch base branch plugin.json once - reused for version bump check and compare link - _base_pjson="" - OLD_VERSION="" - OLD_SOURCE_URL_TMPL="" - if git show "origin/${BASE_REF}:${PLUGIN_JSON}" > /dev/null 2>&1; then - _base_pjson=$(git show "origin/${BASE_REF}:${PLUGIN_JSON}") - OLD_VERSION=$(echo "$_base_pjson" | jq -r '.version // ""') - OLD_SOURCE_URL_TMPL=$(echo "$_base_pjson" | jq -r '.source_url // ""') - fi - - # ── External source checks ──────────────────────────────────────────────────── - source_type=$(jq -r '.source_type // "local"' "$PLUGIN_JSON") - release_link="" - compare_link="" - _gh_tag="" - _old_gh_tag="" - if [[ "$source_type" != "external" ]]; then - _src_count=$(find "$PLUGIN_DIR" -type f \ - ! -name 'plugin.json' ! -name 'README.md' ! -name 'logo.png' \ - | wc -l | tr -d ' ') - if [[ "$_src_count" -eq 0 ]]; then - TABLE_ROWS+=("| Plugin files | ❌ | No source files found in \`plugins/$PLUGIN_NAME/\`. Add your plugin source (e.g. \`main.py\`) or set \`\"source_type\": \"external\"\` in \`plugin.json\` if your plugin is hosted elsewhere |") - failed=1 - fi - fi - if [[ "$source_type" == "external" ]]; then - ext_source_url=$(jq -r '.source_url // ""' "$PLUGIN_JSON") - ext_repo_url=$(jq -r '.repo_url // ""' "$PLUGIN_JSON") - - if [[ -z "$ext_source_url" ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Required when \`source_type\` is \`external\` |") - failed=1 - elif [[ ! "$ext_source_url" =~ ^https:// ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Must be an HTTPS URL |") - failed=1 - elif [[ "$ext_source_url" != *"{version}"* ]]; then - TABLE_ROWS+=("| \`source_url\` | ❌ | Must contain a \`{version}\` placeholder (e.g. \`.../v{version}/plugin.zip\`) |") - failed=1 - else - # TABLE_ROWS+=("| \`source_url\` | ✅ | \`${ext_source_url}\` |") - if [[ $(validate_semver "$VERSION") -eq 1 ]]; then - resolved_url="${ext_source_url//\{version\}/$VERSION}" - http_code=$(curl -o /dev/null -s -w "%{http_code}" --max-time 15 -L "$resolved_url" || echo "000") - if [[ "$http_code" == "200" ]]; then - TABLE_ROWS+=("| Release artifact | ✅ | Artifact reachable at resolved URL |") - if [[ "$resolved_url" =~ ^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/ ]]; then - _gh_owner="${BASH_REMATCH[1]}" - _gh_repo="${BASH_REMATCH[2]}" - _gh_tag="${BASH_REMATCH[3]}" - release_link="https://github.com/${_gh_owner}/${_gh_repo}/releases/tag/${_gh_tag}" - if [[ -n "$OLD_VERSION" && -n "$OLD_SOURCE_URL_TMPL" ]]; then - _old_resolved="${OLD_SOURCE_URL_TMPL//\{version\}/$OLD_VERSION}" - if [[ "$_old_resolved" =~ ^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/ ]]; then - _old_gh_tag="${BASH_REMATCH[1]}" - compare_link="https://github.com/${_gh_owner}/${_gh_repo}/compare/${_old_gh_tag}...${_gh_tag}" - fi - fi - fi - else - TABLE_ROWS+=("| Release artifact | ❌ | Could not reach \`$resolved_url\` (HTTP \`$http_code\`) — ensure the release exists |") - failed=1 - fi - fi - fi - - if [[ -z "$ext_repo_url" ]]; then - TABLE_ROWS+=("| \`repo_url\` | ❌ | Required for external plugins — set to the upstream source repository URL |") - failed=1 - fi - fi - - # Maintainers - if [[ -z "$AUTHOR" ]] && [[ -z "$MAINTAINERS" ]]; then - TABLE_ROWS+=("| Maintainers | ❌ | At least one of \`author\` or \`maintainers\` must include your GitHub username |") - failed=1 - else - DISPLAY_PARTS=() - [[ -n "$AUTHOR" ]] && DISPLAY_PARTS+=("\`$AUTHOR\`") - for m in $MAINTAINERS; do DISPLAY_PARTS+=("\`$m\`"); done - DISPLAY=$(printf '%s, ' "${DISPLAY_PARTS[@]}"); DISPLAY="${DISPLAY%, }" - TABLE_ROWS+=("| Maintainers | ✅ | $DISPLAY |") - fi - - # License (required) - LICENSE_ID=$(jq -r '.license // ""' "$PLUGIN_JSON") - if [[ -z "$LICENSE_ID" ]]; then - TABLE_ROWS+=("| License | ❌ | \`license\` is required - provide an [OSI-approved SPDX identifier](https://spdx.org/licenses/) (e.g. \`MIT\`, \`Apache-2.0\`) |") - failed=1 - else - SPDX_JSON=$(curl -fsSL "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" 2>/dev/null || echo "") - if [[ -z "$SPDX_JSON" ]]; then - TABLE_ROWS+=("| License | ⚠️ | Could not fetch SPDX license list - skipping validation |") - else - SPDX_VALID=$(echo "$SPDX_JSON" | jq --arg lid "$LICENSE_ID" '[.licenses[] | select(.isOsiApproved == true) | .licenseId] | any(. == $lid)') - if [[ "$SPDX_VALID" == "true" ]]; then - LICENSE_NAME=$(echo "$SPDX_JSON" | jq -r --arg lid "$LICENSE_ID" '.licenses[] | select(.licenseId == $lid) | .name') - TABLE_ROWS+=("| License | ✅ | \`$LICENSE_ID\` - $LICENSE_NAME |") - else - TABLE_ROWS+=("| License | ❌ | \`$LICENSE_ID\` is not an [OSI-approved SPDX identifier](https://spdx.org/licenses/) |") - failed=1 - fi - fi - fi - - # ── Access control ──────────────────────────────────────────────────────────── - - # Permission check - use base branch version to prevent self-granting via the PR - IS_REPO_MAINTAINER=$(check_repo_maintainer "$PR_AUTHOR") - if [[ "$IS_REPO_MAINTAINER" -eq 1 ]]; then - TABLE_ROWS+=("| Permission | ✅ | You have permission to modify this plugin |") - has_permission="true" - elif git show "origin/${BASE_REF}:${PLUGIN_JSON}" > /dev/null 2>&1; then - BASE_JSON=$(git show "origin/${BASE_REF}:${PLUGIN_JSON}") - BASE_AUTHOR=$(echo "$BASE_JSON" | jq -r '.author // ""') - BASE_MAINTAINERS=$(echo "$BASE_JSON" | jq -r '[.maintainers[]?] | join(" ")') - if [[ "$PR_AUTHOR" == "$BASE_AUTHOR" ]] || [[ " $BASE_MAINTAINERS " =~ " $PR_AUTHOR " ]]; then - TABLE_ROWS+=("| Permission | ✅ | You have permission to modify this plugin |") - has_permission="true" - else - TABLE_ROWS+=("| Permission | ❌ | \`$PR_AUTHOR\` is not listed in \`author\` or \`maintainers\` |") - failed=1 - fi - else - # New plugin - PR author must list themselves so future PRs can be authorized - if [[ "$PR_AUTHOR" == "$AUTHOR" ]] || [[ " $MAINTAINERS " =~ " $PR_AUTHOR " ]]; then - TABLE_ROWS+=("| Permission | ✅ | New plugin - \`$PR_AUTHOR\` listed in \`author\`/\`maintainers\` |") - has_permission="true" - else - TABLE_ROWS+=("| Permission | ❌ | Add \`\"author\": \"$PR_AUTHOR\"\` to plugin.json |") - failed=1 - fi - fi - - # ── Version checks ──────────────────────────────────────────────────────────── - - # Version format - if [[ $(validate_semver "$VERSION") -eq 1 ]]; then - TABLE_ROWS+=("| Version | ✅ | \`$VERSION\` |") - else - TABLE_ROWS+=("| Version | ❌ | \`$VERSION\` is not valid semver - expected \`X.Y.Z\` |") - failed=1 - fi - - # Version bump - # These fields may be updated without a version bump - they are metadata only - # and do not affect the packaged ZIP artifact. - METADATA_ONLY_FIELDS=("description" "repo_url" "discord_thread" - "min_dispatcharr_version" "max_dispatcharr_version" "deprecated" "unlisted" "maintainers") - - if [[ -n "$_base_pjson" ]]; then - if version_greater_than "$VERSION" "$OLD_VERSION"; then - TABLE_ROWS+=("| Version bump | ✅ | \`$OLD_VERSION\` → \`$VERSION\` |") - else - # Version unchanged - check if every changed field is in the metadata-only allowlist - OLD_JSON="$_base_pjson" - NEW_JSON=$(cat "$PLUGIN_JSON") - - # Produce a newline-separated list of field names that differ (raw strings, no quotes) - changed_fields=$(jq -rn \ - --argjson old "$OLD_JSON" \ - --argjson new "$NEW_JSON" \ - '[$new | keys[]] | map(select($old[.] != $new[.])) | .[]' 2>/dev/null || true) - - metadata_only_change=true - while IFS= read -r field; do - [[ -z "$field" ]] && continue - allowed=false - for mf in "${METADATA_ONLY_FIELDS[@]}"; do - [[ "$field" == "$mf" ]] && allowed=true && break - done - $allowed || { metadata_only_change=false; break; } - done <<< "$changed_fields" - - if $metadata_only_change && [[ -n "$changed_fields" ]]; then - TABLE_ROWS+=("| Version bump | ✅ | \`$OLD_VERSION\` (unchanged - metadata-only update) |") - else - # Any non-metadata plugin.json change or other file change requires a version bump. - # If plugin.json is identical, check whether other files in the directory changed - # so we can give a more accurate error message. - _other_changed="" - if [[ -z "$changed_fields" ]]; then - _other_changed=$(git diff --name-only "origin/${BASE_REF}...HEAD" -- "$PLUGIN_DIR" \ - | grep -v "^${PLUGIN_JSON}$" | head -1 || true) - fi - if [[ -z "$changed_fields" && -z "$_other_changed" ]]; then - TABLE_ROWS+=("| Version bump | ❌ | No changes detected - nothing to publish |") - else - TABLE_ROWS+=("| Version bump | ❌ | \`$VERSION\` must be greater than current \`$OLD_VERSION\` |") - fi - failed=1 - fi - fi - else - TABLE_ROWS+=("| Version bump | ✅ | New plugin |") - is_new="true" - fi - - # Dispatcharr version constraints (optional, hidden if pass) - MIN_DA_VERSION=$(jq -r '.min_dispatcharr_version // ""' "$PLUGIN_JSON") - MAX_DA_VERSION=$(jq -r '.max_dispatcharr_version // ""' "$PLUGIN_JSON") - - if [[ -n "$MIN_DA_VERSION" ]] && [[ $(validate_dispatcharr_version "$MIN_DA_VERSION") -eq 0 ]]; then - TABLE_ROWS+=("| \`min_dispatcharr_version\` | ❌ | \`$MIN_DA_VERSION\` is not valid semver - expected \`X.Y.Z\` or \`vX.Y.Z\` |") - failed=1 - fi - - if [[ -n "$MAX_DA_VERSION" ]] && [[ $(validate_dispatcharr_version "$MAX_DA_VERSION") -eq 0 ]]; then - TABLE_ROWS+=("| \`max_dispatcharr_version\` | ❌ | \`$MAX_DA_VERSION\` is not valid semver - expected \`X.Y.Z\` or \`vX.Y.Z\` |") - failed=1 - fi - - # min/max range sanity (hidden if pass) - if [[ -n "$MIN_DA_VERSION" && -n "$MAX_DA_VERSION" ]]; then - if [[ $(validate_dispatcharr_version "$MAX_DA_VERSION") -eq 1 && $(validate_dispatcharr_version "$MIN_DA_VERSION") -eq 1 ]]; then - _max="${MAX_DA_VERSION#v}" - _min="${MIN_DA_VERSION#v}" - if ! version_greater_than "$_max" "$_min" && [[ "$_max" != "$_min" ]]; then - TABLE_ROWS+=("| Version range | ❌ | \`max_dispatcharr_version\` (\`$MAX_DA_VERSION\`) must be ≥ \`min_dispatcharr_version\` (\`$MIN_DA_VERSION\`) |") - failed=1 - fi - fi - fi - - # ── Optional link fields (hidden if pass) ──────────────────────────────────── - - DISCORD_THREAD=$(jq -r '.discord_thread // ""' "$PLUGIN_JSON") - - if [[ -n "$REPO_URL" ]] && [[ ! "$REPO_URL" =~ ^https?:// ]]; then - TABLE_ROWS+=("| \`repo_url\` | ❌ | Must start with \`http://\` or \`https://\` |") - failed=1 - fi - - if [[ -n "$DISCORD_THREAD" ]] && [[ ! "$DISCORD_THREAD" =~ ^https?:// ]]; then - TABLE_ROWS+=("| \`discord_thread\` | ❌ | Must start with \`http://\` or \`https://\` |") - failed=1 - fi - - print_table - - if [[ -n "$release_link" ]]; then - _link_line="[View release ${_gh_tag} on GitHub](${release_link})" - [[ -n "$compare_link" ]] && _link_line+=" · [Compare ${_old_gh_tag}...${_gh_tag}](${compare_link})" - echo "$_link_line" - echo "" - fi - - # Metadata row (tab-delimited, consumed by aggregate-report.sh) - echo "" - -} > "$OUTPUT_FILE" - - -# Write job outputs -if [[ $failed -eq 0 ]]; then - echo "result=pass" >> "$GITHUB_OUTPUT" -else - echo "result=fail" >> "$GITHUB_OUTPUT" -fi -echo "is_new=$is_new" >> "$GITHUB_OUTPUT" -echo "has_permission=$has_permission" >> "$GITHUB_OUTPUT" - -exit $failed diff --git a/.github/workflows/_clamav-scan.yml b/.github/workflows/_clamav-scan.yml new file mode 100644 index 00000000..1207b95d --- /dev/null +++ b/.github/workflows/_clamav-scan.yml @@ -0,0 +1,182 @@ +name: ClamAV Scan (reusable) + +# Reusable workflow called by validate-plugin.yml. Virus-scans the changed +# plugins of a PR merge commit. pluginctl runs from the base branch (_trusted), +# never from the untrusted merge tree. + +on: + workflow_call: + inputs: + matrix: + required: true + type: string + pr_number: + required: true + type: string + base_ref: + required: true + type: string + outputs: + clamav_status: + value: ${{ jobs.scan.outputs.clamav_status }} + clamav_infected: + value: ${{ jobs.scan.outputs.clamav_infected }} + +permissions: + contents: read + pull-requests: write + issues: write + +env: + PYTHONPATH: ${{ github.workspace }}/_trusted/.github/pluginctl/src + +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + clamav_status: ${{ steps.status.outputs.clamav_status }} + clamav_infected: ${{ steps.status.outputs.clamav_infected }} + steps: + - name: Checkout PR merge commit for scan + uses: actions/checkout@v6 + with: + ref: refs/pull/${{ inputs.pr_number }}/merge + sparse-checkout: .gitignore + sparse-checkout-cone-mode: false + # Merge ref is only virus-scanned by ClamAV; fork code is not executed. + allow-unsafe-pr-checkout: true + + - name: Checkout trusted pluginctl (base) + uses: actions/checkout@v6 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.base_ref }} + path: _trusted + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Limit checkout to changed plugin folders only + run: | + echo '${{ inputs.matrix }}' | jq -r '.[] | "plugins/\(.)"' | git sparse-checkout set --stdin + git checkout + + - name: Get cache keys + id: cache-keys + run: | + echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT + echo "date=$(date +%Y%m%d)" >> $GITHUB_OUTPUT + + - name: Cache ClamAV installation (weekly) + id: cache-clamav-install + uses: actions/cache@v5 + with: + path: /tmp/clamav-apt + key: clamav-install-${{ runner.os }}-${{ steps.cache-keys.outputs.week }} + restore-keys: clamav-install-${{ runner.os }}- + + - name: Cache ClamAV virus definitions (daily) + id: cache-clamav-defs + uses: actions/cache@v5 + with: + path: /tmp/clamav-db + key: clamav-defs-${{ runner.os }}-${{ steps.cache-keys.outputs.date }} + restore-keys: clamav-defs-${{ runner.os }}- + + - name: Install ClamAV + run: | + sudo apt-get update -qq + if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" == 'true' ]]; then + echo "Installing ClamAV from cached packages..." + sudo cp /tmp/clamav-apt/*.deb /var/cache/apt/archives/ 2>/dev/null || true + fi + sudo apt-get install -y --no-install-recommends clamav + if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" != 'true' ]]; then + echo "Saving ClamAV packages to cache..." + mkdir -p /tmp/clamav-apt + find /var/cache/apt/archives/ \( -name 'clamav*.deb' -o -name 'libclamav*.deb' \) \ + -exec cp {} /tmp/clamav-apt/ \; 2>/dev/null || true + fi + + - name: Update virus definitions + run: | + sudo systemctl stop clamav-freshclam 2>/dev/null || true + sudo mkdir -p /tmp/clamav-db + sudo chown -R clamav:clamav /tmp/clamav-db + if [[ "${{ steps.cache-clamav-defs.outputs.cache-hit }}" != 'true' ]]; then + echo "Downloading fresh ClamAV definitions..." + sudo freshclam --datadir=/tmp/clamav-db + else + echo "Using cached ClamAV definitions" + fi + + - name: Scan changed plugin directories + id: scan + run: | + SCAN_TARGETS=() + EXT_SCAN_DIR="/tmp/external-scan" + mkdir -p "$EXT_SCAN_DIR" + while IFS= read -r plugin_name; do + plugin_json="plugins/$plugin_name/plugin.json" + [[ ! -f "$plugin_json" ]] && continue + source_type=$(jq -r '.source_type // "local"' "$plugin_json") + if [[ "$source_type" == "external" ]]; then + version=$(jq -r '.version' "$plugin_json") + source_url_template=$(jq -r '.source_url // ""' "$plugin_json") + source_url="${source_url_template//\{version\}/$version}" + echo "Downloading external ZIP for ClamAV scan: $source_url" + scan_dir="$EXT_SCAN_DIR/$plugin_name" + mkdir -p "$scan_dir/extracted" + zip_file="$scan_dir/${plugin_name}.zip" + if curl -fsSL "$source_url" -o "$zip_file" --max-time 60; then + unzip -q "$zip_file" -d "$scan_dir/extracted" || true + SCAN_TARGETS+=("$scan_dir/extracted") + else + echo "::warning::Could not download $source_url, scanning plugin.json only for $plugin_name" + SCAN_TARGETS+=("plugins/$plugin_name") + fi + else + SCAN_TARGETS+=("plugins/$plugin_name") + fi + done < <(echo '${{ inputs.matrix }}' | jq -r '.[]') + echo "Scanning: ${SCAN_TARGETS[*]}" + set +e + clamscan --database=/tmp/clamav-db \ + --recursive --infected --no-summary \ + "${SCAN_TARGETS[@]}" > clamav-output.txt 2>&1 + echo "exit_code=$?" >> $GITHUB_OUTPUT + set -e + cat clamav-output.txt + + - name: Parse ClamAV output and set status + id: status + if: always() + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + run: python -m pluginctl clamav-report --output clamav-output.txt --scan-exit "${{ steps.scan.outputs.exit_code }}" + + - name: Upload ClamAV findings for PR comment + if: always() && steps.status.outputs.clamav_status == 'failure' + uses: actions/upload-artifact@v7 + with: + name: clamav-findings + path: clamav-findings.md + if-no-files-found: ignore + + - name: Apply quarantine label + if: always() && steps.status.outputs.clamav_status == 'failure' + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + gh label create "QUARANTINE" \ + --color "FFFF00" \ + --description "ClamAV detected a potential threat in this PR" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + gh pr edit "${{ inputs.pr_number }}" \ + --add-label "QUARANTINE" \ + --repo "$GITHUB_REPOSITORY" + + - name: Fail job if ClamAV detected threats + if: always() && steps.status.outputs.clamav_status == 'failure' + run: exit 1 diff --git a/.github/workflows/_codeql-scan.yml b/.github/workflows/_codeql-scan.yml new file mode 100644 index 00000000..1b052180 --- /dev/null +++ b/.github/workflows/_codeql-scan.yml @@ -0,0 +1,158 @@ +name: CodeQL Scan (reusable) + +# Reusable workflow called by validate-plugin.yml. Runs CodeQL over the changed +# plugins of a PR merge commit and turns the SARIF into PR-comment inputs. +# pluginctl always runs from the base branch (checked out into _trusted), never +# from the untrusted merge tree. + +on: + workflow_call: + inputs: + matrix: + required: true + type: string + pr_number: + required: true + type: string + base_ref: + required: true + type: string + outputs: + codeql_status: + value: ${{ jobs.analyze.outputs.codeql_status }} + codeql_errors: + value: ${{ jobs.analyze.outputs.codeql_errors }} + codeql_warnings: + value: ${{ jobs.analyze.outputs.codeql_warnings }} + codeql_mediums: + value: ${{ jobs.analyze.outputs.codeql_mediums }} + codeql_lows: + value: ${{ jobs.analyze.outputs.codeql_lows }} + codeql_unscanned_langs: + value: ${{ jobs.analyze.outputs.codeql_unscanned_langs }} + +permissions: + contents: read + security-events: write + +env: + PYTHONPATH: ${{ github.workspace }}/_trusted/.github/pluginctl/src + +jobs: + analyze: + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + codeql_status: ${{ steps.status.outputs.codeql_status }} + codeql_errors: ${{ steps.status.outputs.codeql_errors }} + codeql_warnings: ${{ steps.status.outputs.codeql_warnings }} + codeql_mediums: ${{ steps.status.outputs.codeql_mediums }} + codeql_lows: ${{ steps.status.outputs.codeql_lows }} + codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} + steps: + - name: Checkout PR merge commit for analysis + uses: actions/checkout@v6 + with: + ref: refs/pull/${{ inputs.pr_number }}/merge + sparse-checkout: .gitignore + sparse-checkout-cone-mode: false + # Merge ref is analyzed statically by CodeQL only; fork code is not executed. + allow-unsafe-pr-checkout: true + + - name: Checkout trusted pluginctl (base) + uses: actions/checkout@v6 + with: + repository: ${{ github.repository }} + ref: ${{ inputs.base_ref }} + path: _trusted + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Limit checkout to changed plugin folders only + run: | + echo '${{ inputs.matrix }}' | jq -r '.[] | "plugins/\(.)"' | git sparse-checkout set --stdin + git checkout + + - name: Populate external plugin source for analysis + run: | + while IFS= read -r plugin_name; do + plugin_json="plugins/$plugin_name/plugin.json" + [[ ! -f "$plugin_json" ]] && continue + source_type=$(jq -r '.source_type // "local"' "$plugin_json") + [[ "$source_type" != "external" ]] && continue + version=$(jq -r '.version' "$plugin_json") + source_url_template=$(jq -r '.source_url // ""' "$plugin_json") + source_url="${source_url_template//\{version\}/$version}" + echo "Fetching release ZIP for CodeQL: $source_url" + curl -fsSL "$source_url" -o "/tmp/${plugin_name}-release.zip" + mkdir -p "/tmp/${plugin_name}-src" + python3 -c "import zipfile,os; z=zipfile.ZipFile('/tmp/${plugin_name}-release.zip'); d='/tmp/${plugin_name}-src'; [z.extract((setattr(m,'filename',m.filename.replace(chr(92),'/')) or m),d) for m in z.infolist()]" + cp -rn "/tmp/${plugin_name}-src/." "plugins/$plugin_name/" + rm -rf "/tmp/${plugin_name}-release.zip" "/tmp/${plugin_name}-src" + done < <(echo '${{ inputs.matrix }}' | jq -r '.[]') + + - name: Detect supported languages + id: detect-langs + run: python -m pluginctl detect-langs + + - name: Initialize CodeQL + if: steps.detect-langs.outputs.found == 'true' + uses: github/codeql-action/init@v4 + with: + languages: ${{ steps.detect-langs.outputs.languages }} + build-mode: none + queries: security-extended + + - name: Perform CodeQL Analysis + if: steps.detect-langs.outputs.found == 'true' + id: analyze + uses: github/codeql-action/analyze@v4 + with: + category: pr-${{ inputs.pr_number }} + output: sarif-results + upload: false + continue-on-error: true + + - name: Parse SARIF and set CodeQL status + id: status + if: always() + env: + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + python -m pluginctl sarif \ + --results-dir sarif-results \ + --repo "${{ github.repository }}" \ + --sha "$(git rev-parse HEAD)" \ + --matrix '${{ inputs.matrix }}' \ + --analyze-outcome "${{ steps.analyze.outcome }}" \ + --found "${{ steps.detect-langs.outputs.found }}" \ + --languages "${{ steps.detect-langs.outputs.languages }}" \ + --unscanned-langs "${{ steps.detect-langs.outputs.unscanned_langs }}" + + - name: Upload findings detail for PR comment + if: always() && steps.status.outputs.codeql_status == 'failure' + uses: actions/upload-artifact@v7 + with: + name: codeql-findings + path: codeql-findings.md + if-no-files-found: ignore + + - name: Upload medium findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-medium-findings + path: codeql-medium-findings.md + if-no-files-found: ignore + + - name: Upload low findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-low-findings + path: codeql-low-findings.md + if-no-files-found: ignore + + - name: Fail job if CodeQL found high/error/critical issues + if: always() && steps.status.outputs.codeql_status == 'failure' + run: exit 1 diff --git a/.github/workflows/auto-merge-updates.yml b/.github/workflows/auto-merge-updates.yml index 3329c1ee..51d38417 100644 --- a/.github/workflows/auto-merge-updates.yml +++ b/.github/workflows/auto-merge-updates.yml @@ -1,18 +1,19 @@ name: Auto-merge Plugin Updates # Opt-in via the AUTO_MERGE_UPDATES repo variable. When enabled, squash-merges -# a PR automatically once "Validate Plugin" succeeds - but ONLY for pure -# plugin version-bump updates. Never for new plugins, repo/tooling changes, -# invalid PRs, or PRs that mix a new plugin with an update. +# a PR automatically once "Validate Plugin" succeeds, but ONLY for pure plugin +# version-bump updates. Never for new plugins, repo/tooling changes, invalid +# PRs, or PRs that mix a new plugin with an update. # # Runs the merge with the GitHub App token specifically (never GITHUB_TOKEN): # pushes authored by the App's bot identity correctly cascade into -# publish-plugins.yml's `on: push` trigger, whereas GITHUB_TOKEN-authored -# pushes are exempt from triggering other workflows. If the App isn't -# configured, auto-merge is skipped entirely rather than falling back. +# publish-plugins.yml's `on: push` trigger, whereas GITHUB_TOKEN-authored pushes +# are exempt from triggering other workflows. If the App isn't configured, +# auto-merge is skipped entirely rather than falling back. env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: workflow_run: @@ -33,150 +34,32 @@ jobs: group: auto-merge-updates-${{ github.event.workflow_run.head_sha }} cancel-in-progress: false steps: - - name: Checkout base branch scripts (trusted) + - name: Checkout base branch (trusted) uses: actions/checkout@v6 with: repository: ${{ github.repository }} ref: main fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - name: Skip - GitHub App not configured - if: steps.config.outputs.use_app != 'true' + if: steps.token.outputs.use_app != 'true' run: | echo "::notice::GH_APP_ID or GH_APP_PRIVATE_KEY not configured. Skipping auto-merge (merging with GITHUB_TOKEN would not trigger publish-plugins.yml)." - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - - name: Resolve PR for this workflow run - if: steps.config.outputs.use_app == 'true' - id: resolve-pr + - name: Auto-merge pure plugin update + if: steps.token.outputs.use_app == 'true' env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GH_REPO: ${{ github.repository }} + GITHUB_REPOSITORY: ${{ github.repository }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} - run: | - # workflow_run.pull_requests is unreliable for pull_request_target runs - # from forks, so match on head SHA against open PRs instead. - MATCHES=$(gh pr list --state open --json number,headRefOid,isDraft \ - | jq -c --arg sha "$HEAD_SHA" '[.[] | select(.headRefOid == $sha)]') - COUNT=$(echo "$MATCHES" | jq 'length') - - if [[ "$COUNT" -ne 1 ]]; then - echo "::notice::Found $COUNT open PR(s) matching head SHA $HEAD_SHA - skipping (expected exactly 1)." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - PR_NUMBER=$(echo "$MATCHES" | jq -r '.[0].number') - IS_DRAFT=$(echo "$MATCHES" | jq -r '.[0].isDraft') - - if [[ "$IS_DRAFT" == "true" ]]; then - echo "::notice::PR #$PR_NUMBER is a draft - skipping." - echo "found=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "found=true" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" - - - name: Check labels and mergeable state - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' - id: check-labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - INFO=$(gh pr view "$PR_NUMBER" --json labels,mergeable,mergeStateStatus) - LABELS=$(echo "$INFO" | jq -r '[.labels[].name] | join(",")') - MERGEABLE=$(echo "$INFO" | jq -r '.mergeable') - - echo "Labels: $LABELS" - echo "Mergeable: $MERGEABLE" - - has_label() { - echo ",$LABELS," | grep -qF ",$1," - } - - OK=true - has_label "Plugin Update" || { echo "::notice::Missing 'Plugin Update' label - skipping."; OK=false; } - has_label "New Plugin" && { echo "::notice::Has 'New Plugin' label - skipping."; OK=false; } - has_label "Repo Update" && { echo "::notice::Has 'Repo Update' label - skipping."; OK=false; } - has_label "Invalid" && { echo "::notice::Has 'Invalid' label - skipping."; OK=false; } - has_label "QUARANTINE" && { echo "::notice::Has 'QUARANTINE' label - skipping."; OK=false; } - has_label "CodeQL Suppression Used" && { echo "::notice::PR uses a CodeQL suppression comment - skipping auto-merge, requires maintainer review."; OK=false; } - [[ "$MERGEABLE" == "MERGEABLE" ]] || { echo "::notice::PR not cleanly mergeable (mergeable=$MERGEABLE) - skipping."; OK=false; } - - echo "ok=$OK" >> "$GITHUB_OUTPUT" - - - name: Independently re-verify this is a pure plugin update - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' && steps.check-labels.outputs.ok == 'true' - id: reverify - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - # Defense in depth: labels are applied by a trusted job, but they are - # ordinary PR labels and could in principle be hand-edited by anyone - # with write access. Re-derive the same signal directly from the diff - # rather than trusting labels alone. - git fetch origin "pull/${PR_NUMBER}/merge:pr-merge-${PR_NUMBER}" 2>/dev/null \ - || { echo "::notice::Could not fetch merge ref for PR #$PR_NUMBER - skipping."; echo "ok=false" >> "$GITHUB_OUTPUT"; exit 0; } - git checkout -q "pr-merge-${PR_NUMBER}" - - MERGE_BASE=$(git merge-base origin/main HEAD) - CHANGED=$(git diff --name-only "$MERGE_BASE" HEAD) - - OK=true - - OUTSIDE=$(echo "$CHANGED" | grep -v '^plugins/' || true) - if [[ -n "$OUTSIDE" ]]; then - echo "::notice::Changes outside plugins/ detected - skipping. Files: $OUTSIDE" - OK=false - fi - - PLUGIN_SLUGS=$(echo "$CHANGED" | grep '^plugins/' | cut -d'/' -f2 | sort -u) - if [[ -z "$PLUGIN_SLUGS" ]]; then - echo "::notice::No plugin changes detected in diff - skipping." - OK=false - fi - - while IFS= read -r slug; do - [[ -z "$slug" ]] && continue - if ! git show "origin/main:plugins/${slug}/plugin.json" > /dev/null 2>&1; then - echo "::notice::Plugin '$slug' does not exist on main - this is a new plugin, not an update. Skipping." - OK=false - fi - done <<< "$PLUGIN_SLUGS" - - echo "ok=$OK" >> "$GITHUB_OUTPUT" - - - name: Squash-merge PR - if: steps.config.outputs.use_app == 'true' && steps.resolve-pr.outputs.found == 'true' && steps.check-labels.outputs.ok == 'true' && steps.reverify.outputs.ok == 'true' - env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ steps.resolve-pr.outputs.pr_number }} - run: | - gh pr merge "$PR_NUMBER" --squash --delete-branch=false - echo "Auto-merged PR #$PR_NUMBER" + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} + run: python -m pluginctl automerge --head-sha "$HEAD_SHA" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0b037181..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: CodeQL - -# Pull request analysis is handled by validate-plugin.yml (codeql-analyze job), -# which runs CodeQL as part of the plugin validation flow. -on: - workflow_dispatch: - # push: - # branches: [main] - # schedule: - # - cron: '0 6 * * 1' - -jobs: - analyze: - name: Analyze (python) - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Initialize CodeQL - uses: github/codeql-action/init@v4 - with: - languages: python - queries: security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v4 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 - with: - category: /language:python \ No newline at end of file diff --git a/.github/workflows/publish-plugins.yml b/.github/workflows/publish-plugins.yml index dce61b22..f1bcb173 100644 --- a/.github/workflows/publish-plugins.yml +++ b/.github/workflows/publish-plugins.yml @@ -5,6 +5,7 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: push: @@ -13,12 +14,7 @@ on: paths: - 'plugins/**' - '.github/workflows/publish-plugins.yml' - - '.github/scripts/publish/run.sh' - - '.github/scripts/publish/build-zips.sh' - - '.github/scripts/publish/plugin-readmes.sh' - - '.github/scripts/publish/cleanup.sh' - - '.github/scripts/publish/generate-manifest.sh' - - '.github/scripts/publish/releases-readme.sh' + - '.github/pluginctl/**' - '.github/scripts/keys/**' workflow_dispatch: inputs: @@ -28,7 +24,7 @@ on: default: false required: false force_rebuild_plugin: - description: 'Optional: plugin slug to target (e.g. dispatcharr-exporter). Only that plugin is cleared and rebuilt - all others are untouched. Leave blank to wipe and rebuild the entire releases branch (WARNING: purges all version history).' + description: 'Optional: plugin slug to target (e.g. dispatcharr-exporter). Only that plugin is cleared and rebuilt, all others are untouched. Leave blank to wipe and rebuild the entire releases branch (WARNING: purges all version history).' type: string default: '' required: false @@ -52,53 +48,33 @@ jobs: with: fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - - name: Make scripts executable - run: chmod +x .github/scripts/publish/*.sh - - name: Publish plugins to releases branch id: publish env: - GITHUB_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + APP_SLUG: ${{ steps.token.outputs.app-slug }} GITHUB_REPOSITORY: ${{ github.repository }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} FORCE_REBUILD: ${{ inputs.force_rebuild }} FORCE_REBUILD_CONFIRM: ${{ inputs.force_rebuild_confirm }} FORCE_REBUILD_PLUGIN: ${{ inputs.force_rebuild_plugin }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | set -o pipefail if [[ "${FORCE_REBUILD}" == "true" && "${FORCE_REBUILD_CONFIRM}" != "CONFIRM" ]]; then echo "::error::force_rebuild requires typing CONFIRM in the force_rebuild_confirm input." exit 1 fi - .github/scripts/publish/run.sh ${{ github.ref_name }} 2>&1 | tee publish.log + python -m pluginctl publish ${{ github.ref_name }} 2>&1 | tee publish.log - name: Generate job summary if: always() @@ -107,17 +83,9 @@ jobs: if [ -f publish.log ]; then if grep -q "Successfully published" publish.log; then echo "✅ Publishing completed successfully" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - # Extract plugin info from the temporary repo if available - if [ -d /tmp/tmp.*/repo ]; then - REPO_DIR=$(find /tmp -maxdepth 1 -name "tmp.*" -type d 2>/dev/null | head -1)/repo - if [ -f "$REPO_DIR/manifest.json" ]; then - cat "$REPO_DIR/manifest.json" | jq -r '.manifest.plugins[] | "- **\(.name)** v\(.latest_version)"' >> $GITHUB_STEP_SUMMARY - fi - fi else echo "❌ Publishing failed" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "See logs for details." >> $GITHUB_STEP_SUMMARY fi - fi \ No newline at end of file + fi diff --git a/.github/workflows/update-external-readme.yml b/.github/workflows/update-external-readme.yml index 83417701..7a22aba3 100644 --- a/.github/workflows/update-external-readme.yml +++ b/.github/workflows/update-external-readme.yml @@ -3,6 +3,9 @@ name: Publish Plugin Docs PR # Reads the README.md generated by the Publish Plugins workflow from the # releases branch and opens (or updates) a PR in a target repository. +env: + PYTHONPATH: .github/pluginctl/src + on: workflow_run: workflows: ["Publish Plugins"] @@ -17,9 +20,8 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest timeout-minutes: 10 - steps: - - name: Load and validate configuration + - name: Validate configuration id: config env: GH_APP_ID: ${{ vars.GH_APP_ID }} @@ -31,176 +33,37 @@ jobs: echo "::notice::GH_APP_ID, GH_APP_PRIVATE_KEY, EXTERNAL_README_REPO, or EXTERNAL_README_PATH not configured - skipping." echo "skip=true" >> "$GITHUB_OUTPUT" else - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "app_id=$GH_APP_ID" >> "$GITHUB_OUTPUT" - echo "target_repo=$EXTERNAL_README_REPO" >> "$GITHUB_OUTPUT" - echo "target_path=$EXTERNAL_README_PATH" >> "$GITHUB_OUTPUT" + echo "skip=false" >> "$GITHUB_OUTPUT" fi - - name: Generate GitHub App token - if: steps.config.outputs.skip != 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - # Token is org-scoped; the app must be installed on the target repo - owner: ${{ github.repository_owner }} - - - name: Checkout releases branch README + # Keep the base branch checked out: the composite action and pluginctl live + # here. The releases-branch README is read via the origin/releases ref below. + - name: Checkout base branch (trusted) if: steps.config.outputs.skip != 'true' uses: actions/checkout@v6 with: - ref: releases fetch-depth: 1 - clean: false - sparse-checkout: README.md - sparse-checkout-cone-mode: false - - name: Extract release metadata from commit + - name: GitHub App token if: steps.config.outputs.skip != 'true' - id: meta - run: | - COMMIT_MSG=$(git log -1 --format="%B" HEAD) - - # Extract source commit SHA from "Source commit: " line - SOURCE_COMMIT=$(echo "$COMMIT_MSG" | grep -oP '(?<=Source commit: )[0-9a-f]+' || true) - - # Extract plugin list - lines starting with "- " after the source commit line - PLUGIN_LIST=$(echo "$COMMIT_MSG" | grep -E '^\- [a-z0-9].*@' || true) - - RELEASES_COMMIT=$(git rev-parse --short HEAD) - - echo "source_commit=${SOURCE_COMMIT}" >> "$GITHUB_OUTPUT" - echo "releases_commit=${RELEASES_COMMIT}" >> "$GITHUB_OUTPUT" - - # Write plugin list to a file to avoid quoting issues in multi-line output - printf '%s' "$PLUGIN_LIST" > /tmp/plugin_list.txt + id: token + uses: ./.github/actions/gh-app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} - - name: Strip releases-branch preamble from README + - name: Fetch releases branch if: steps.config.outputs.skip != 'true' - env: - SOURCE_REPO: ${{ github.repository }} - run: | - # Remove the intro paragraph and Quick Access section that are only - # relevant on the releases branch itself, keeping everything from - # "## Available Plugins" onward (preserving the "# Plugin Releases" title). - CONTRIB_URL="https://github.com/${SOURCE_REPO}/blob/main/CONTRIBUTING.md" - awk -v contrib="$CONTRIB_URL" ' - /^# Plugin Releases/ { print; print ""; print "Want to get your plugin added to this list? Check out the [plugin repository](" contrib ") to learn how to contribute."; next } - /^## Available Plugins/ { found=1 } - found { print } - ' README.md > README.stripped.md - mv README.stripped.md README.md - - # Rewrite relative links (./zips/...) to absolute GitHub URLs pointing - # at the releases branch. Anchor links (#...) are unaffected because - # they never start with "./". - BASE_URL="https://github.com/${SOURCE_REPO}/tree/releases" - sed -i "s|](\./|](${BASE_URL}/|g" README.md + run: git fetch origin releases - - name: Create or update PR in target repo + - name: Create or update external README PR if: steps.config.outputs.skip != 'true' env: - TARGET_REPO: ${{ steps.config.outputs.target_repo }} - TARGET_PATH: ${{ steps.config.outputs.target_path }} - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} + TARGET_REPO: ${{ vars.EXTERNAL_README_REPO }} + TARGET_PATH: ${{ vars.EXTERNAL_README_PATH }} SOURCE_REPO: ${{ github.repository }} - SOURCE_COMMIT: ${{ steps.meta.outputs.source_commit }} - RELEASES_COMMIT: ${{ steps.meta.outputs.releases_commit }} + GITHUB_REPOSITORY: ${{ github.repository }} PR_REVIEWER: ${{ vars.PR_REVIEWER }} - run: | - if [[ ! -f README.md ]]; then - echo "::error::README.md not found on the releases branch - has the Publish Plugins workflow run yet?" - exit 1 - fi - - PLUGIN_LIST=$(cat /tmp/plugin_list.txt) - - # Build a human-readable change summary block - build_summary() { - echo "**Source commit:** [\`${SOURCE_COMMIT}\`](https://github.com/${SOURCE_REPO}/commit/${SOURCE_COMMIT})" - echo "**Releases commit:** [\`${RELEASES_COMMIT}\`](https://github.com/${SOURCE_REPO}/commit/${RELEASES_COMMIT})" - if [[ -n "$PLUGIN_LIST" ]]; then - echo "" - echo "**Plugins updated:**" - while IFS= read -r line; do - echo "$line" - done <<< "$PLUGIN_LIST" - fi - } - - BRANCH="auto/dispatcharr-plugin-readme" - DEFAULT_BRANCH=$(gh api "repos/$TARGET_REPO" --jq '.default_branch') - - # Create branch if it does not already exist - if ! gh api "repos/$TARGET_REPO/branches/$BRANCH" > /dev/null 2>&1; then - BASE_SHA=$(gh api "repos/$TARGET_REPO/git/refs/heads/$DEFAULT_BRANCH" \ - --jq '.object.sha') - gh api "repos/$TARGET_REPO/git/refs" \ - -X POST \ - -f ref="refs/heads/$BRANCH" \ - -f sha="$BASE_SHA" - echo "Created branch $BRANCH" - fi - - # Get the existing file SHA and content from the branch - EXISTING_RESPONSE=$(gh api "repos/$TARGET_REPO/contents/$TARGET_PATH?ref=$BRANCH" 2>/dev/null || true) - FILE_SHA=$(echo "$EXISTING_RESPONSE" | jq -r '.sha // empty') - - # Skip if the only change is the timestamp line at the bottom of the README - if [[ -n "$FILE_SHA" ]]; then - EXISTING_CONTENT=$(echo "$EXISTING_RESPONSE" | jq -r '.content' | base64 -d 2>/dev/null || true) - NEW_STRIPPED=$(grep -v '^\*Last updated:' README.md || true) - EXISTING_STRIPPED=$(echo "$EXISTING_CONTENT" | grep -v '^\*Last updated:' || true) - if [[ "$NEW_STRIPPED" == "$EXISTING_STRIPPED" ]]; then - echo "README content unchanged (only timestamp differs) - skipping." - echo "::notice::External README not updated: only the timestamp line changed." - exit 0 - fi - fi - - # Upload file (base64-encoded content is required by the GitHub Contents API) - PUT_ARGS=( - -X PUT - -f message="chore: update plugin releases listing (source commit $SOURCE_COMMIT)" - -f content="$(base64 -w 0 README.md)" - -f branch="$BRANCH" - ) - [[ -n "$FILE_SHA" ]] && PUT_ARGS+=(-f sha="$FILE_SHA") - - gh api "repos/$TARGET_REPO/contents/$TARGET_PATH" "${PUT_ARGS[@]}" - - # Re-use an existing open PR for this branch; open a new one if none exists - EXISTING_PR=$(gh pr list \ - --repo "$TARGET_REPO" \ - --state open \ - --head "$BRANCH" \ - --json number \ - | jq -r '.[0].number // empty') - - if [[ -n "$EXISTING_PR" ]]; then - # Add a comment to the existing PR summarising this update - COMMENT=$(printf '### Plugin README update\n\n%s' "$(build_summary)") - gh pr comment "$EXISTING_PR" --repo "$TARGET_REPO" --body "$COMMENT" - echo "Added update comment to existing PR #$EXISTING_PR" - else - PR_BODY=$(printf 'Automated update of the plugin releases listings generated from [`%s`](https://github.com/%s).\n\n%s' \ - "$SOURCE_REPO" "$SOURCE_REPO" "$(build_summary)") - REVIEWER_ARGS=() - if [[ -n "${PR_REVIEWER:-}" ]]; then - IFS=',' read -ra _reviewers <<< "$PR_REVIEWER" - for _r in "${_reviewers[@]}"; do - _r="${_r#@}"; _r="${_r// /}" # strip @ and whitespace - [[ -n "$_r" ]] && REVIEWER_ARGS+=(--reviewer "$_r") - done - fi - gh pr create \ - --repo "$TARGET_REPO" \ - --head "$BRANCH" \ - --base "$DEFAULT_BRANCH" \ - --title "chore: update plugin releases listings" \ - --body "$PR_BODY" \ - "${REVIEWER_ARGS[@]}" - echo "Created new PR" - fi + run: python -m pluginctl external-readme diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 4d0a977c..9b19ddce 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -8,8 +8,9 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src # Comma-separated lists of blocked GitHub usernames and plugin slugs. - # Set these in repo Settings → Variables as AUTHOR_BLACKLIST and PLUGIN_BLACKLIST. + # Set these in repo Settings -> Variables as AUTHOR_BLACKLIST and PLUGIN_BLACKLIST. AUTHOR_BLACKLIST: ${{ vars.AUTHOR_BLACKLIST }} PLUGIN_BLACKLIST: ${{ vars.PLUGIN_BLACKLIST }} @@ -27,7 +28,7 @@ concurrency: jobs: # -------------------------------------------------------------------------- - # Job 1: Post a notice on draft PRs - no validation runs yet + # Job 1: Post a notice on draft PRs, no validation runs yet # -------------------------------------------------------------------------- draft-notice: if: github.event.pull_request.draft == true @@ -39,40 +40,21 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Post draft notice env: PR_NUMBER: ${{ github.event.pull_request.number }} - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - BOT_LOGIN: ${{ steps.config.outputs.use_app == 'true' && format('{0}[bot]', steps.app-token.outputs.app-slug) || 'github-actions[bot]' }} + GH_TOKEN: ${{ steps.token.outputs.token }} + BOT_LOGIN: ${{ steps.token.outputs.bot-login }} GH_REPO: ${{ github.repository }} run: | MARKER="" @@ -86,7 +68,7 @@ jobs: fi # -------------------------------------------------------------------------- - # Job 2: Detect which plugins changed and build the matrix + # Job 2: Detect which plugins changed and build the matrix (+ blacklist) # -------------------------------------------------------------------------- detect-changes: if: github.event.pull_request.draft == false @@ -95,8 +77,8 @@ jobs: outputs: matrix: ${{ steps.detect.outputs.matrix }} plugin_count: ${{ steps.detect.outputs.plugin_count }} - close_pr: ${{ steps.blacklist.outputs.close_pr || steps.detect.outputs.close_pr }} - close_reason: ${{ steps.blacklist.outputs.close_reason || steps.detect.outputs.close_reason }} + close_pr: ${{ steps.detect.outputs.close_pr }} + close_reason: ${{ steps.detect.outputs.close_reason }} outside_files: ${{ steps.detect.outputs.outside_files }} outside_violation: ${{ steps.detect.outputs.outside_violation }} skip_validation: ${{ steps.detect.outputs.skip_validation }} @@ -104,132 +86,31 @@ jobs: has_new_plugin: ${{ steps.detect.outputs.has_new_plugin }} has_updated_plugin: ${{ steps.detect.outputs.has_updated_plugin }} steps: - - name: Checkout base branch scripts (trusted) - uses: actions/checkout@v6 + - name: Trusted checkout + uses: ./.github/actions/trusted-checkout with: + base-ref: ${{ github.event.pull_request.base.ref }} + head-repo: ${{ github.event.pull_request.head.repo.full_name }} + head-sha: ${{ github.event.pull_request.head.sha }} repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.ref }} - fetch-depth: 0 - sparse-checkout: .github/scripts - sparse-checkout-cone-mode: false - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} - - - name: Save trusted scripts before fork checkout - run: cp -r .github/scripts /tmp/trusted-scripts - - - name: Checkout PR plugins (untrusted content only) - uses: actions/checkout@v6 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - sparse-checkout: plugins - sparse-checkout-cone-mode: false - clean: false - # Fork content is only read as data (sparse plugins/), never executed; - # trusted scripts are saved/restored around this step. - allow-unsafe-pr-checkout: true - - - name: Restore trusted scripts - run: mkdir -p .github && cp -r /tmp/trusted-scripts .github/scripts - - name: Re-fetch base branch refs - run: git fetch https://github.com/${{ github.repository }} +${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Detect changed plugins id: detect env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} run: | - chmod +x .github/scripts/validate/*.sh - .github/scripts/validate/detect-changes.sh \ - "${{ github.event.pull_request.user.login }}" \ - "${{ github.event.pull_request.base.ref }}" \ - "${{ github.event.pull_request.head.ref }}" - - - name: Check author and plugin blacklists - id: blacklist - # Only run if detect didn't already decide to close (avoids redundant output) - if: steps.detect.outputs.close_pr != 'true' - env: - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - MATRIX: ${{ steps.detect.outputs.matrix }} - run: | - AUTHOR_BL="${AUTHOR_BLACKLIST:-}" - PLUGIN_BL="${PLUGIN_BLACKLIST:-}" - - # No-op if neither list is configured - if [[ -z "$AUTHOR_BL" && -z "$PLUGIN_BL" ]]; then - exit 0 - fi - - MATCHED=false - REASON="" - - # Check author blacklist (case-insensitive, strips whitespace around commas) - if [[ -n "$AUTHOR_BL" ]]; then - IFS=',' read -ra BLOCKED <<< "$AUTHOR_BL" - for entry in "${BLOCKED[@]}"; do - slug="${entry// /}" - if [[ "${PR_AUTHOR,,}" == "${slug,,}" ]]; then - MATCHED=true - REASON="author-blacklisted" - echo "::warning::PR author '$PR_AUTHOR' is on the author blacklist." - break - fi - done - fi - - # Check plugin blacklist (case-insensitive) - if [[ "$MATCHED" != "true" && -n "$PLUGIN_BL" ]]; then - IFS=',' read -ra BLOCKED <<< "$PLUGIN_BL" - while IFS= read -r plugin; do - [[ -z "$plugin" ]] && continue - for entry in "${BLOCKED[@]}"; do - slug="${entry// /}" - if [[ "${plugin,,}" == "${slug,,}" ]]; then - MATCHED=true - REASON="plugin-blacklisted" - echo "::warning::Plugin '$plugin' is on the plugin blacklist." - break 2 - fi - done - done < <(printf '%s' "$MATRIX" | jq -r '.[]' 2>/dev/null || true) - fi - - if [[ "$MATCHED" == "true" ]]; then - echo "close_pr=true" >> "$GITHUB_OUTPUT" - echo "close_reason=$REASON" >> "$GITHUB_OUTPUT" - fi + python -m pluginctl detect \ + --author "${{ github.event.pull_request.user.login }}" \ + --base-ref "${{ github.event.pull_request.base.ref }}" \ + --head-ref "${{ github.event.pull_request.head.ref }}" # -------------------------------------------------------------------------- # Job 3: Apply PR labels based on change classification @@ -245,75 +126,31 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Apply labels env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - HAS_NEW_PLUGIN: ${{ needs.detect-changes.outputs.has_new_plugin }} - HAS_UPDATED_PLUGIN: ${{ needs.detect-changes.outputs.has_updated_plugin }} - OUTSIDE_FILES: ${{ needs.detect-changes.outputs.outside_files }} - OUTSIDE_VIOLATION: ${{ needs.detect-changes.outputs.outside_violation }} - CLOSE_PR: ${{ needs.detect-changes.outputs.close_pr }} run: | - apply_label() { - local label=$1 condition=$2 - if [[ "$condition" == "true" ]]; then - gh pr edit "$PR_NUMBER" --add-label "$label" || true - else - gh pr edit "$PR_NUMBER" --remove-label "$label" 2>/dev/null || true - fi - } - apply_label "New Plugin" "$HAS_NEW_PLUGIN" - apply_label "Plugin Update" "$HAS_UPDATED_PLUGIN" - # Repo Update only when there are outside files AND it's not a violation (authorized) - if [[ -n "$OUTSIDE_FILES" && "$OUTSIDE_VIOLATION" != "true" ]]; then - IS_REPO_UPDATE=true - else - IS_REPO_UPDATE=false - fi - apply_label "Repo Update" "$IS_REPO_UPDATE" - # Invalid when PR has unauthorized outside changes or unauthorized plugin modifications - if [[ "$OUTSIDE_VIOLATION" == "true" || "$CLOSE_PR" == "true" ]]; then - IS_INVALID=true - else - IS_INVALID=false - fi - apply_label "Invalid" "$IS_INVALID" + python -m pluginctl label \ + --pr "${{ github.event.pull_request.number }}" \ + --has-new-plugin "${{ needs.detect-changes.outputs.has_new_plugin }}" \ + --has-updated-plugin "${{ needs.detect-changes.outputs.has_updated_plugin }}" \ + --outside-files "${{ needs.detect-changes.outputs.outside_files }}" \ + --outside-violation "${{ needs.detect-changes.outputs.outside_violation }}" \ + --close-pr "${{ needs.detect-changes.outputs.close_pr }}" # -------------------------------------------------------------------------- # Job 4: Validate PR title format - # Runs on every non-draft, non-closing PR event (including 'edited', which - # fires when the title is renamed). # -------------------------------------------------------------------------- validate-title: needs: [detect-changes] @@ -325,796 +162,46 @@ jobs: title_feedback: ${{ steps.check.outputs.title_feedback }} title_suggestion: ${{ steps.check.outputs.title_suggestion }} steps: + - name: Checkout config + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 1 + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + - name: Validate PR title format id: check - env: - PR_TITLE: ${{ github.event.pull_request.title }} - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - PLUGIN_COUNT: ${{ needs.detect-changes.outputs.plugin_count }} - MATRIX: ${{ needs.detect-changes.outputs.matrix }} - SKIP_VALIDATION: ${{ needs.detect-changes.outputs.skip_validation }} run: | - TITLE="$PR_TITLE" - AUTHOR="$PR_AUTHOR" - COUNT="${PLUGIN_COUNT:-0}" - SKIP="${SKIP_VALIDATION:-false}" - MATRIX_JSON="${MATRIX:-[]}" - TITLE_VALID=true - TITLE_FEEDBACK="" - TITLE_SUGGESTION="" - - # Build a context-appropriate suggestion for this PR - if [[ "$COUNT" == "0" ]]; then - CONTEXT_SUGGESTION="[repo]: Brief description of changes" - elif [[ "$COUNT" == "1" ]]; then - SLUG=$(printf '%s' "$MATRIX_JSON" | jq -r '.[0] // "plugin-slug"' || echo "plugin-slug") - CONTEXT_SUGGESTION="[$SLUG]: Brief description of changes" - else - CONTEXT_SUGGESTION="[$AUTHOR]: Brief description of changes" - fi - - # Check format and extract prefix using bash =~ to avoid grep/sed ERE inconsistencies. - # [^]] inside bash =~ means "not ]" (] is literal when first after [^). - if [[ "$TITLE" =~ ^\[([^]]+)\]:?[[:space:]]+.+ ]]; then - PREFIX="${BASH_REMATCH[1]}" - - # Validate prefix matches expected context - if [[ "$COUNT" == "0" ]]; then - if [[ "$PREFIX" != "repo" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For repo-level or non-plugin changes, the prefix should be \`[repo]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - elif [[ "$COUNT" == "1" ]]; then - EXPECTED=$(printf '%s' "$MATRIX_JSON" | jq -r '.[0] // ""' || echo "") - if [[ -n "$EXPECTED" && "$PREFIX" != "$EXPECTED" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For a single plugin change, the prefix should match the plugin folder name: \`[$EXPECTED]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - else - if [[ "$PREFIX" != "$AUTHOR" ]]; then - TITLE_VALID=false - TITLE_FEEDBACK="For changes to multiple plugins, the prefix should be your GitHub username: \`[$AUTHOR]\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - fi - else - TITLE_VALID=false - TITLE_FEEDBACK="PR title does not match the required format. Expected: \`[prefix] description\`." - TITLE_SUGGESTION="$CONTEXT_SUGGESTION" - fi - - echo "title_valid=$TITLE_VALID" >> "$GITHUB_OUTPUT" - { - echo "title_feedback<> "$GITHUB_OUTPUT" - { - echo "title_suggestion<> "$GITHUB_OUTPUT" - - if [[ "$TITLE_VALID" != "true" ]]; then - echo "::error::PR title does not match the required format. See CONTRIBUTING.md for details." - exit 1 - fi + python -m pluginctl check-title \ + --title "${{ github.event.pull_request.title }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --matrix '${{ needs.detect-changes.outputs.matrix }}' # -------------------------------------------------------------------------- - # Job 5: CodeQL security and quality analysis (runs after validate-plugin) + # Job 5: CodeQL security and quality analysis (reusable workflow) # -------------------------------------------------------------------------- codeql-analyze: needs: [detect-changes, validate-plugin, validate-title] if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.validate-title.outputs.title_valid == 'true' - runs-on: ubuntu-latest - timeout-minutes: 15 - outputs: - codeql_status: ${{ steps.status.outputs.codeql_status }} - codeql_errors: ${{ steps.status.outputs.codeql_errors }} - codeql_warnings: ${{ steps.status.outputs.codeql_warnings }} - codeql_mediums: ${{ steps.status.outputs.codeql_mediums }} - codeql_lows: ${{ steps.status.outputs.codeql_lows }} - codeql_suppressed: ${{ steps.status.outputs.codeql_suppressed }} - codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} - steps: - - name: Checkout PR merge commit for analysis - uses: actions/checkout@v6 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - # Minimal placeholder - overridden immediately by the next step - # (actions/checkout doesn't support dynamic expressions in sparse-checkout) - sparse-checkout: .gitignore - sparse-checkout-cone-mode: false - # Merge ref is analyzed statically by CodeQL only; fork code is not executed. - allow-unsafe-pr-checkout: true - - - name: Limit checkout to changed plugin folders only - run: | - { - echo '.github/codeql' - echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[] | "plugins/\(.)"' - } | git sparse-checkout set --stdin - git checkout - - - name: Populate external plugin source for analysis - run: | - while IFS= read -r plugin_name; do - plugin_json="plugins/$plugin_name/plugin.json" - [[ ! -f "$plugin_json" ]] && continue - - source_type=$(jq -r '.source_type // "local"' "$plugin_json") - [[ "$source_type" != "external" ]] && continue - - version=$(jq -r '.version' "$plugin_json") - source_url_template=$(jq -r '.source_url // ""' "$plugin_json") - source_url="${source_url_template//\{version\}/$version}" - - echo "Fetching release ZIP for CodeQL: $source_url" - curl -fsSL "$source_url" -o "/tmp/${plugin_name}-release.zip" - - mkdir -p "/tmp/${plugin_name}-src" - python3 -c "import zipfile,os; z=zipfile.ZipFile('/tmp/${plugin_name}-release.zip'); d='/tmp/${plugin_name}-src'; [z.extract((setattr(m,'filename',m.filename.replace(chr(92),'/')) or m),d) for m in z.infolist()]" - # Copy extracted files into plugins dir without overwriting the registry plugin.json - cp -rn "/tmp/${plugin_name}-src/." "plugins/$plugin_name/" - rm -rf "/tmp/${plugin_name}-release.zip" "/tmp/${plugin_name}-src" - echo "Release ZIP for $plugin_name extracted into plugins/$plugin_name/" - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - - name: Detect supported languages - id: detect-langs - run: | - LANGS=() - UNSCANNED=() - - # --- CodeQL-supported languages --- - if find plugins -name '*.py' -print -quit | grep -q .; then - LANGS+=(python) - fi - if find plugins \ - \( -path '*/node_modules/*' -o -path '*/dist/*' -o -path '*/build/*' -o -path '*/static/*' \) -prune -o \ - \( -name '*.js' -o -name '*.ts' -o -name '*.jsx' -o -name '*.tsx' \) -print \ - -quit | grep -q .; then - LANGS+=(javascript) - fi - if find plugins -name '*.go' -print -quit | grep -q .; then - LANGS+=(go) - fi - if find plugins -name '*.rb' -print -quit | grep -q .; then - LANGS+=(ruby) - fi - if find plugins \( -name '*.java' -o -name '*.kt' -o -name '*.kts' \) -print -quit | grep -q .; then - LANGS+=("java-kotlin") - fi - if find plugins \( -name '*.c' -o -name '*.cpp' -o -name '*.cc' -o -name '*.h' -o -name '*.hpp' \) -print -quit | grep -q .; then - LANGS+=("c-cpp") - fi - - # --- Files present but not supported by CodeQL --- - if find plugins \( -name '*.sh' -o -name '*.bash' \) -print -quit | grep -q .; then - UNSCANNED+=(shell) - fi - if find plugins -name '*.php' -print -quit | grep -q .; then - UNSCANNED+=(php) - fi - if find plugins -name '*.lua' -print -quit | grep -q .; then - UNSCANNED+=(lua) - fi - if find plugins \( -name '*.pl' -o -name '*.pm' \) -print -quit | grep -q .; then - UNSCANNED+=(perl) - fi - if find plugins -name '*.rs' -print -quit | grep -q .; then - UNSCANNED+=(rust) - fi - - UNSCANNED_CSV="" - [[ ${#UNSCANNED[@]} -gt 0 ]] && UNSCANNED_CSV=$(IFS=,; echo "${UNSCANNED[*]}") - echo "unscanned_langs=$UNSCANNED_CSV" >> "$GITHUB_OUTPUT" - - if [[ ${#LANGS[@]} -gt 0 ]]; then - LANG_CSV=$(IFS=,; echo "${LANGS[*]}") - echo "found=true" >> "$GITHUB_OUTPUT" - echo "languages=$LANG_CSV" >> "$GITHUB_OUTPUT" - echo "Detected languages for CodeQL: $LANG_CSV" - if [[ -n "$UNSCANNED_CSV" ]]; then echo "Unscanned (no CodeQL support): $UNSCANNED_CSV"; fi - else - echo "found=false" >> "$GITHUB_OUTPUT" - echo "languages=" >> "$GITHUB_OUTPUT" - echo "No supported language files found in changed plugins - skipping CodeQL." - if [[ -n "$UNSCANNED_CSV" ]]; then echo "Unscanned file types detected (no CodeQL support): $UNSCANNED_CSV"; fi - fi - - - name: Get merge commit SHA - id: merge - run: echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: Initialize CodeQL - if: steps.detect-langs.outputs.found == 'true' - uses: github/codeql-action/init@v4 - with: - languages: ${{ steps.detect-langs.outputs.languages }} - build-mode: none - # Query suite (security-extended: all security severities, no quality queries) and - # per-language alert-suppression packs (so inline `codeql[]` comments work) - # both live in this config file - see .github/codeql/codeql-config.yml for why. - config-file: .github/codeql/codeql-config.yml - - - name: Perform CodeQL Analysis - if: steps.detect-langs.outputs.found == 'true' - id: analyze - uses: github/codeql-action/analyze@v4 - with: - category: pr-${{ github.event.pull_request.number }} - output: sarif-results - upload: false - continue-on-error: true - - - name: Set CodeQL status output - id: status - if: always() - run: | - # Only block on security findings with CVSS score >= 7.0 (HIGH or CRITICAL). - # CodeQL stores this in rule properties["security-severity"], not in result.level. - # A result with a non-empty .suppressions array was recognized by CodeQL's own - # engine as validly suppressed (e.g. a correctly-placed inline `codeql[...]` - # comment) - trust that signal and never let it count as blocking/medium/low. - JQ_BLOCKING=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - ((.ruleId // .rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) - ] | length' - JQ_MEDIUM=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - ((.ruleId // .rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev >= 6.0 and $sev < 7.0) - ] | length' - JQ_LOW=' - [ .runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - ((.ruleId // .rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev < 6.0) - ] | length' - JQ_SUPPRESSED=' - [ .runs[] | (.results // [])[] | select((.suppressions // []) | length > 0) ] | length' - RESULT_COUNT=0 - MEDIUM_COUNT=0 - LOW_COUNT=0 - SUPPRESSED_COUNT=0 - TOTAL_COUNT=0 - if [[ -d "sarif-results" ]]; then - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - CONTENT=$(gunzip -c "$f") - else - CONTENT=$(cat "$f") - fi - COUNT=$(echo "$CONTENT" | jq "$JQ_BLOCKING") - MED=$(echo "$CONTENT" | jq "$JQ_MEDIUM") - LOW=$(echo "$CONTENT" | jq "$JQ_LOW") - SUPPRESSED=$(echo "$CONTENT" | jq "$JQ_SUPPRESSED") - TOT=$(echo "$CONTENT" | jq '[.runs[] | (.results // [])[]] | length') - echo "File $f: ${COUNT:-0} blocking, ${MED:-0} medium, ${LOW:-0} low, ${SUPPRESSED:-0} suppressed, $TOT total" - echo "$CONTENT" | jq -r \ - '[ .runs[] | . as $run | - ([ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] | from_entries) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - ((.ruleId // .rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) | - " [blocking] \($rid) sec-sev=\($secmap[$rid] // "n/a")" ] | .[]' || true - RESULT_COUNT=$((RESULT_COUNT + ${COUNT:-0})) - MEDIUM_COUNT=$((MEDIUM_COUNT + ${MED:-0})) - LOW_COUNT=$((LOW_COUNT + ${LOW:-0})) - SUPPRESSED_COUNT=$((SUPPRESSED_COUNT + ${SUPPRESSED:-0})) - TOTAL_COUNT=$((TOTAL_COUNT + ${TOT:-0})) - done - fi - WARN_COUNT=$(( TOTAL_COUNT > RESULT_COUNT ? TOTAL_COUNT - RESULT_COUNT : 0 )) - echo "Found $RESULT_COUNT high/critical, $MEDIUM_COUNT medium, $LOW_COUNT low, $SUPPRESSED_COUNT suppressed, and $WARN_COUNT other CodeQL result(s)" - echo "codeql_errors=$RESULT_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_warnings=$WARN_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_mediums=$MEDIUM_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_lows=$LOW_COUNT" >> "$GITHUB_OUTPUT" - echo "codeql_suppressed=$SUPPRESSED_COUNT" >> "$GITHUB_OUTPUT" - - # Build list of external plugin path prefixes so findings links are suppressed - # for files that came from a downloaded ZIP and don't exist in this repo. - EXTERNAL_PREFIXES='[]' - while IFS= read -r _plugin_name; do - _pjson="plugins/$_plugin_name/plugin.json" - [[ -f "$_pjson" ]] || continue - _stype=$(jq -r '.source_type // "local"' "$_pjson" 2>/dev/null || echo "local") - if [[ "$_stype" == "external" ]]; then - EXTERNAL_PREFIXES=$(printf '%s' "$EXTERNAL_PREFIXES" | jq --arg p "plugins/$_plugin_name/" '. + [$p]') - fi - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - # Generate a findings detail file for inclusion in the PR comment - if [[ "$RESULT_COUNT" -gt 0 ]]; then - MERGE_SHA=$(git rev-parse HEAD) - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - select((($secmap[$rid] // "0") | tonumber) >= 7.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") - # Data-flow queries can repeat the same sentence once per source/flow reaching - # the same sink - collapse repeats while preserving first-seen order. - | [splits("(?<=[.!?]) ")] as $sentences - | reduce $sentences[] as $s ([]; if any(.[]; . == $s) then . else . + [$s] end) - | join(" ")) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-findings.md - fi - # Generate medium findings detail file for informational display in the PR comment - if [[ "$MEDIUM_COUNT" -gt 0 ]]; then - MERGE_SHA=${MERGE_SHA:-$(git rev-parse HEAD)} - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev >= 6.0 and $sev < 7.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") - # Data-flow queries can repeat the same sentence once per source/flow reaching - # the same sink - collapse repeats while preserving first-seen order. - | [splits("(?<=[.!?]) ")] as $sentences - | reduce $sentences[] as $s ([]; if any(.[]; . == $s) then . else . + [$s] end) - | join(" ")) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-medium-findings.md - fi - # Generate low findings detail file for informational display in the PR comment (collapsed) - if [[ "$LOW_COUNT" -gt 0 ]]; then - MERGE_SHA=${MERGE_SHA:-$(git rev-parse HEAD)} - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ( - [ (($run.tool.driver.rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}), - ((($run.tool.extensions // [])[].rules // [])[] | {key: (.id // ""), value: ((.properties["security-severity"] // "0") | tostring)}) ] - | from_entries - ) as $secmap | - ($run.results // [])[] | - select((.suppressions // []) | length == 0) | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - (($secmap[$rid] // "0") | tonumber) as $sev | - select($sev < 6.0) | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") - # Data-flow queries can repeat the same sentence once per source/flow reaching - # the same sink - collapse repeats while preserving first-seen order. - | [splits("(?<=[.!?]) ")] as $sentences - | reduce $sentences[] as $s ([]; if any(.[]; . == $s) then . else . + [$s] end) - | join(" ")) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-low-findings.md - fi - # Generate suppressed findings detail file for informational display in the PR comment. - # These were excluded from the blocking/medium/low tables above because CodeQL's own - # engine recognized and applied an inline suppression comment. - if [[ "$SUPPRESSED_COUNT" -gt 0 ]]; then - MERGE_SHA=${MERGE_SHA:-$(git rev-parse HEAD)} - { - echo "| Rule | Location | Description |" - echo "|------|----------|-------------|" - for f in sarif-results/*.sarif sarif-results/*.sarif.gz; do - [[ -f "$f" ]] || continue - if [[ "$f" == *.gz ]]; then - FC=$(gunzip -c "$f") - else - FC=$(cat "$f") - fi - echo "$FC" | jq -r \ - --arg repo "$GITHUB_REPOSITORY" \ - --arg sha "$MERGE_SHA" \ - --argjson external_prefixes "$EXTERNAL_PREFIXES" \ - '.runs[] | - . as $run | - ($run.results // [])[] | - select((.suppressions // []) | length > 0) | - . as $result | - (($result.ruleId // $result.rule.id // "") | tostring) as $rid | - (.locations[0].physicalLocation.artifactLocation.uri // "?") as $uri | - ((.locations[0].physicalLocation.region.startLine // "?") | tostring) as $line | - (.message.text // "no description" | gsub("\n"; " ") | gsub("://"; "\u200b://") | gsub("www\\."; "www\u200b.") | gsub("#(?=[0-9])"; "#\u200b") | gsub("\\[(?[^\\]]+)\\][(][0-9]+[)]"; .c) | gsub("\\["; "[") | gsub("\\]"; "]") - # Data-flow queries can repeat the same sentence once per source/flow reaching - # the same sink - collapse repeats while preserving first-seen order. - | [splits("(?<=[.!?]) ")] as $sentences - | reduce $sentences[] as $s ([]; if any(.[]; . == $s) then . else . + [$s] end) - | join(" ")) as $msg | - ([$external_prefixes[] | . as $p | $uri | startswith($p)] | any) as $is_external | - (if ($uri != "?" and $line != "?" and ($is_external | not)) then "[\($uri):\($line)](https://github.com/\($repo)/blob/\($sha)/\($uri)#L\($line))" else "\($uri):\($line)" end) as $loc | - "| `\($rid)` | \($loc) | \($msg) |"' - done - } > codeql-suppressed-findings.md - fi - ANALYZE_FAILED=false - if [[ "${{ steps.detect-langs.outputs.found }}" == 'true' && "${{ steps.analyze.outcome }}" != "success" && "${{ steps.analyze.outcome }}" != "" ]]; then - ANALYZE_FAILED=true - fi - - UNSCANNED_CSV="${{ steps.detect-langs.outputs.unscanned_langs }}" - CONFIG_ERROR_LANGS="" - - # CodeQL sets CODEQL_ACTION_JOB_STATUS=JOB_STATUS_CONFIGURATION_ERROR when it found - # no indexable source for a requested language (e.g. a committed file that's only - # vendored/minified/generated code, which CodeQL's own extractor refuses to treat as - # source). That's not a security finding or a broken analysis - don't block the PR on - # it, just note which language(s) went unscanned. Any other non-success outcome (a - # real extractor crash, OOM, etc.) still fails the job below. - if [[ "$ANALYZE_FAILED" == 'true' && "${CODEQL_ACTION_JOB_STATUS:-}" == 'JOB_STATUS_CONFIGURATION_ERROR' ]]; then - echo "::warning::CodeQL reported a configuration error (no indexable source found) for language(s): ${{ steps.detect-langs.outputs.languages }}. Treating as skipped instead of failing the PR." - CONFIG_ERROR_LANGS="codeql-config-error($(echo '${{ steps.detect-langs.outputs.languages }}' | tr ',' '+'))" - UNSCANNED_CSV="${UNSCANNED_CSV:+$UNSCANNED_CSV,}${CONFIG_ERROR_LANGS}" - ANALYZE_FAILED=false - fi - - if [[ "$RESULT_COUNT" -gt 0 || "$ANALYZE_FAILED" == 'true' ]]; then - echo "codeql_status=failure" >> "$GITHUB_OUTPUT" - elif [[ "${{ steps.detect-langs.outputs.found }}" == 'false' || -n "$CONFIG_ERROR_LANGS" ]]; then - echo "codeql_status=skipped" >> "$GITHUB_OUTPUT" - else - echo "codeql_status=success" >> "$GITHUB_OUTPUT" - fi - echo "codeql_unscanned_langs=$UNSCANNED_CSV" >> "$GITHUB_OUTPUT" - - - name: Upload findings detail for PR comment - if: always() && steps.status.outputs.codeql_status == 'failure' - uses: actions/upload-artifact@v7 - with: - name: codeql-findings - path: codeql-findings.md - if-no-files-found: ignore - - - name: Upload medium findings detail for PR comment - if: always() - uses: actions/upload-artifact@v7 - with: - name: codeql-medium-findings - path: codeql-medium-findings.md - if-no-files-found: ignore - - - name: Upload low findings detail for PR comment - if: always() - uses: actions/upload-artifact@v7 - with: - name: codeql-low-findings - path: codeql-low-findings.md - if-no-files-found: ignore - - - name: Upload suppressed findings detail for PR comment - if: always() - uses: actions/upload-artifact@v7 - with: - name: codeql-suppressed-findings - path: codeql-suppressed-findings.md - if-no-files-found: ignore - - - name: Apply/clear CodeQL suppression label - # Purely informational for auto-merge gating (see auto-merge-updates.yml) - never - # fails the job, so a maintainer can still merge by hand after reviewing. - if: always() - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - SUPPRESSED_COUNT: ${{ steps.status.outputs.codeql_suppressed }} - run: | - gh label create "CodeQL Suppression Used" \ - --color "FBCA04" \ - --description "PR relies on an inline CodeQL suppression comment - requires maintainer review" \ - --repo "$GH_REPO" 2>/dev/null || true - if [[ "${SUPPRESSED_COUNT:-0}" -gt 0 ]]; then - gh pr edit "$PR_NUMBER" --add-label "CodeQL Suppression Used" --repo "$GH_REPO" - else - gh pr edit "$PR_NUMBER" --remove-label "CodeQL Suppression Used" --repo "$GH_REPO" 2>/dev/null || true - fi - - - name: Fail job if CodeQL found high/error/critical issues - if: always() && steps.status.outputs.codeql_status == 'failure' - run: exit 1 + uses: ./.github/workflows/_codeql-scan.yml + with: + matrix: ${{ needs.detect-changes.outputs.matrix }} + pr_number: ${{ github.event.pull_request.number }} + base_ref: ${{ github.event.pull_request.base.ref }} # -------------------------------------------------------------------------- - # Job 6: ClamAV antivirus scan (runs after validate-plugin) + # Job 6: ClamAV antivirus scan (reusable workflow) # -------------------------------------------------------------------------- clamav-scan: needs: [detect-changes, validate-plugin, validate-title] if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.validate-title.outputs.title_valid == 'true' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - clamav_status: ${{ steps.status.outputs.clamav_status }} - clamav_infected: ${{ steps.status.outputs.clamav_infected }} - steps: - - name: Checkout PR merge commit for scan - uses: actions/checkout@v6 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - sparse-checkout: .gitignore - sparse-checkout-cone-mode: false - # Merge ref is only virus-scanned by ClamAV; fork code is not executed. - allow-unsafe-pr-checkout: true - - - name: Limit checkout to changed plugin folders only - run: | - echo '${{ needs.detect-changes.outputs.matrix }}' \ - | jq -r '.[] | "plugins/\(.)"' \ - | git sparse-checkout set --stdin - git checkout - - - name: Get cache keys - id: cache-keys - run: | - echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT - echo "date=$(date +%Y%m%d)" >> $GITHUB_OUTPUT - - - name: Cache ClamAV installation (weekly) - id: cache-clamav-install - uses: actions/cache@v5 - with: - path: /tmp/clamav-apt - key: clamav-install-${{ runner.os }}-${{ steps.cache-keys.outputs.week }} - restore-keys: clamav-install-${{ runner.os }}- - - - name: Cache ClamAV virus definitions (daily) - id: cache-clamav-defs - uses: actions/cache@v5 - with: - path: /tmp/clamav-db - key: clamav-defs-${{ runner.os }}-${{ steps.cache-keys.outputs.date }} - restore-keys: clamav-defs-${{ runner.os }}- - - - name: Install ClamAV - run: | - sudo apt-get update -qq - if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" == 'true' ]]; then - echo "Installing ClamAV from cached packages..." - sudo cp /tmp/clamav-apt/*.deb /var/cache/apt/archives/ 2>/dev/null || true - fi - sudo apt-get install -y --no-install-recommends clamav - if [[ "${{ steps.cache-clamav-install.outputs.cache-hit }}" != 'true' ]]; then - echo "Saving ClamAV packages to cache..." - mkdir -p /tmp/clamav-apt - find /var/cache/apt/archives/ \( -name 'clamav*.deb' -o -name 'libclamav*.deb' \) \ - -exec cp {} /tmp/clamav-apt/ \; 2>/dev/null || true - fi - - - name: Update virus definitions - run: | - sudo systemctl stop clamav-freshclam 2>/dev/null || true - sudo mkdir -p /tmp/clamav-db - sudo chown -R clamav:clamav /tmp/clamav-db - if [[ "${{ steps.cache-clamav-defs.outputs.cache-hit }}" != 'true' ]]; then - echo "Downloading fresh ClamAV definitions..." - sudo freshclam --datadir=/tmp/clamav-db - else - echo "Using cached ClamAV definitions" - fi - - - name: Scan changed plugin directories - id: scan - run: | - SCAN_TARGETS=() - EXT_SCAN_DIR="/tmp/external-scan" - mkdir -p "$EXT_SCAN_DIR" - - while IFS= read -r plugin_name; do - plugin_json="plugins/$plugin_name/plugin.json" - [[ ! -f "$plugin_json" ]] && continue - - source_type=$(jq -r '.source_type // "local"' "$plugin_json") - - if [[ "$source_type" == "external" ]]; then - version=$(jq -r '.version' "$plugin_json") - source_url_template=$(jq -r '.source_url // ""' "$plugin_json") - source_url="${source_url_template//\{version\}/$version}" - - echo "Downloading external ZIP for ClamAV scan: $source_url" - scan_dir="$EXT_SCAN_DIR/$plugin_name" - mkdir -p "$scan_dir/extracted" - zip_file="$scan_dir/${plugin_name}.zip" - - if curl -fsSL "$source_url" -o "$zip_file" --max-time 60; then - unzip -q "$zip_file" -d "$scan_dir/extracted" || true - SCAN_TARGETS+=("$scan_dir/extracted") - else - echo "::warning::Could not download $source_url — scanning plugin.json only for $plugin_name" - SCAN_TARGETS+=("plugins/$plugin_name") - fi - else - SCAN_TARGETS+=("plugins/$plugin_name") - fi - done < <(echo '${{ needs.detect-changes.outputs.matrix }}' | jq -r '.[]') - - echo "Scanning: ${SCAN_TARGETS[*]}" - set +e - clamscan --database=/tmp/clamav-db \ - --recursive --infected --no-summary \ - "${SCAN_TARGETS[@]}" > clamav-output.txt 2>&1 - echo "exit_code=$?" >> $GITHUB_OUTPUT - set -e - cat clamav-output.txt - - - name: Set ClamAV status outputs - id: status - if: always() - env: - SCAN_EXIT: ${{ steps.scan.outputs.exit_code }} - run: | - INFECTED=0 - if [[ -f "clamav-output.txt" ]]; then - INFECTED=$(grep -c ' FOUND$' clamav-output.txt || true) - fi - echo "clamav_infected=$INFECTED" >> "$GITHUB_OUTPUT" - - if [[ "$INFECTED" -gt 0 ]]; then - { - echo "| File | Signature |" - echo "|------|-----------|" - grep ' FOUND$' clamav-output.txt | while IFS= read -r line; do - FULL_PATH=$(echo "$line" | sed 's/: .* FOUND$//') - FILE=$(echo "$FULL_PATH" | sed "s|^${GITHUB_WORKSPACE}/||") - SIG=$(echo "$line" | sed 's/^[^:]*: //; s/ FOUND$//') - HASH=$(sha256sum "$FULL_PATH" 2>/dev/null | cut -d' ' -f1 || true) - if [[ -n "$HASH" ]]; then - echo "| \`$FILE\` | [\`$SIG\`](https://www.virustotal.com/gui/file/${HASH}) |" - else - echo "| \`$FILE\` | \`$SIG\` |" - fi - done - } > clamav-findings.md - echo "clamav_status=failure" >> "$GITHUB_OUTPUT" - elif [[ "${SCAN_EXIT:-0}" -ge 2 ]]; then - echo "clamav_status=failure" >> "$GITHUB_OUTPUT" - else - echo "clamav_status=success" >> "$GITHUB_OUTPUT" - fi - - - name: Upload ClamAV findings for PR comment - if: always() && steps.status.outputs.clamav_status == 'failure' - uses: actions/upload-artifact@v7 - with: - name: clamav-findings - path: clamav-findings.md - if-no-files-found: ignore - - - name: Load app ID from config - if: always() && steps.status.outputs.clamav_status == 'failure' - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: always() && steps.status.outputs.clamav_status == 'failure' && steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 - with: - client-id: ${{ steps.config.outputs.app_id }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - - name: Apply quarantine label - if: always() && steps.status.outputs.clamav_status == 'failure' - env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - run: | - gh label create "QUARANTINE" \ - --color "FFFF00" \ - --description "ClamAV detected a potential threat in this PR" \ - --repo "$GITHUB_REPOSITORY" 2>/dev/null || true - gh pr edit "${{ github.event.pull_request.number }}" \ - --add-label "QUARANTINE" \ - --repo "$GITHUB_REPOSITORY" - - - name: Fail job if ClamAV detected threats - if: always() && steps.status.outputs.clamav_status == 'failure' - run: exit 1 + uses: ./.github/workflows/_clamav-scan.yml + with: + matrix: ${{ needs.detect-changes.outputs.matrix }} + pr_number: ${{ github.event.pull_request.number }} + base_ref: ${{ github.event.pull_request.base.ref }} # -------------------------------------------------------------------------- # Job 7: Validate each plugin in parallel via matrix @@ -1129,79 +216,33 @@ jobs: matrix: plugin: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: - - name: Checkout base branch scripts (trusted) - uses: actions/checkout@v6 + - name: Trusted checkout + uses: ./.github/actions/trusted-checkout with: + base-ref: ${{ github.event.pull_request.base.ref }} + head-repo: ${{ github.event.pull_request.head.repo.full_name }} + head-sha: ${{ github.event.pull_request.head.sha }} repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.ref }} - fetch-depth: 0 - sparse-checkout: .github/scripts - sparse-checkout-cone-mode: false - - - name: Fetch base branch - run: git fetch origin ${{ github.event.pull_request.base.ref }} - - - name: Save trusted scripts before fork checkout - run: cp -r .github/scripts /tmp/trusted-scripts - - - name: Checkout PR plugins (untrusted content only) - uses: actions/checkout@v6 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 0 - sparse-checkout: plugins - sparse-checkout-cone-mode: false - clean: false - # Fork content is only read as data (sparse plugins/), never executed; - # trusted scripts are saved/restored around this step. - allow-unsafe-pr-checkout: true - - - name: Restore trusted scripts - run: mkdir -p .github && cp -r /tmp/trusted-scripts .github/scripts - - - name: Re-fetch base branch refs - run: git fetch https://github.com/${{ github.repository }} +${{ github.event.pull_request.base.ref }}:refs/remotes/origin/${{ github.event.pull_request.base.ref }} - - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Validate ${{ matrix.plugin }} id: validate env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} run: | - chmod +x .github/scripts/validate/*.sh set +e - .github/scripts/validate/validate.sh \ - "${{ matrix.plugin }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ github.event.pull_request.base.ref }}" \ - "${{ matrix.plugin }}.fragment.md" + python -m pluginctl validate \ + --plugin "${{ matrix.plugin }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --base-ref "${{ github.event.pull_request.base.ref }}" \ + --out "${{ matrix.plugin }}.fragment.md" echo "exit_code=$?" >> $GITHUB_OUTPUT continue-on-error: true @@ -1225,40 +266,21 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout scripts + - name: Checkout config uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Download all report fragments uses: actions/download-artifact@v8 with: @@ -1288,13 +310,6 @@ jobs: path: codeql-low-findings continue-on-error: true - - name: Download CodeQL suppressed findings detail - uses: actions/download-artifact@v8 - with: - name: codeql-suppressed-findings - path: codeql-suppressed-findings - continue-on-error: true - - name: Download ClamAV findings detail uses: actions/download-artifact@v8 with: @@ -1305,7 +320,7 @@ jobs: - name: Aggregate and post comment id: report env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} DISCORD_URL: ${{ vars.DISCORD_URL }} CODEQL_RESULT: ${{ needs.codeql-analyze.outputs.codeql_status }} @@ -1313,7 +328,6 @@ jobs: CODEQL_WARNINGS: ${{ needs.codeql-analyze.outputs.codeql_warnings }} CODEQL_MEDIUMS: ${{ needs.codeql-analyze.outputs.codeql_mediums }} CODEQL_LOWS: ${{ needs.codeql-analyze.outputs.codeql_lows }} - CODEQL_SUPPRESSED: ${{ needs.codeql-analyze.outputs.codeql_suppressed }} CODEQL_UNSCANNED_LANGS: ${{ needs.codeql-analyze.outputs.codeql_unscanned_langs }} CLAMAV_RESULT: ${{ needs.clamav-scan.outputs.clamav_status }} CLAMAV_INFECTED: ${{ needs.clamav-scan.outputs.clamav_infected }} @@ -1324,15 +338,16 @@ jobs: TITLE_FEEDBACK: ${{ needs.validate-title.outputs.title_feedback }} TITLE_SUGGESTION: ${{ needs.validate-title.outputs.title_suggestion }} BASE_REF: ${{ github.event.pull_request.base.ref }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | - chmod +x .github/scripts/validate/*.sh set +e - .github/scripts/validate/report.sh \ - "${{ github.event.pull_request.number }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ needs.detect-changes.outputs.plugin_count }}" \ - "false" \ - "fragments" + python -m pluginctl report \ + --pr "${{ github.event.pull_request.number }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --close-pr false \ + --fragments-dir fragments echo "exit_code=$?" >> $GITHUB_OUTPUT - name: Fail workflow if any plugin failed @@ -1350,60 +365,39 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout scripts + - name: Checkout config uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.base.ref }} fetch-depth: 1 - sparse-checkout: .github/scripts + sparse-checkout: .github sparse-checkout-cone-mode: false - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - name: Post close comment and close PR env: - GH_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} + GH_TOKEN: ${{ steps.token.outputs.token }} GITHUB_REPOSITORY: ${{ github.repository }} DISCORD_URL: ${{ vars.DISCORD_URL }} CLOSE_REASON: ${{ needs.detect-changes.outputs.close_reason }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | - chmod +x .github/scripts/validate/*.sh - .github/scripts/validate/report.sh \ - "${{ github.event.pull_request.number }}" \ - "${{ github.event.pull_request.user.login }}" \ - "${{ needs.detect-changes.outputs.plugin_count }}" \ - "true" \ - "/dev/null" + python -m pluginctl report \ + --pr "${{ github.event.pull_request.number }}" \ + --author "${{ github.event.pull_request.user.login }}" \ + --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ + --close-pr true \ + --fragments-dir /dev/null # -------------------------------------------------------------------------- - # Job 10: Gate - single fixed status check for branch protection rules - # Reference this job by name "Plugin PR Check" in your branch protection. - # Passes only when all jobs succeed. Fails for any error, including - # unauthorized PRs. + # Job 10: Gate, single fixed status check for branch protection rules # -------------------------------------------------------------------------- plugin-pr-check: name: Plugin PR Check @@ -1412,6 +406,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: + - name: Checkout config + uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.ref }} + fetch-depth: 1 + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + - name: Check for quarantine label env: GH_TOKEN: ${{ github.token }} @@ -1437,41 +439,4 @@ jobs: CLAMAV_STATUS: ${{ needs.clamav-scan.outputs.clamav_status }} VALIDATE_RESULT: ${{ needs.validate-plugin.result }} REPORT_RESULT: ${{ needs.report.result }} - run: | - if [[ "$DETECT_RESULT" != "success" ]]; then - echo "::error::Plugin detection failed or no plugin changes found." - exit 1 - fi - if [[ "$SKIP_VALIDATION" == "true" ]]; then - echo "No plugin changes detected and author has write access - passing." - exit 0 - fi - if [[ "$TITLE_RESULT" == "failure" ]]; then - echo "::error::PR title does not match the required format. Rename the PR and re-run." - exit 1 - fi - if [[ "$OUTSIDE_VIOLATION" == "true" ]]; then - echo "::error::PR contains unauthorized changes outside the plugins/ directory." - exit 1 - fi - if [[ "$CLOSE_PR" == "true" ]]; then - echo "::error::PR is unauthorized - no permission to modify these plugins." - exit 1 - fi - if [[ "$CODEQL_RESULT" == "failure" || "$CODEQL_STATUS" == "failure" ]]; then - echo "::error::CodeQL security analysis failed. See the Security tab for details." - exit 1 - fi - if [[ "$CLAMAV_RESULT" == "failure" || "$CLAMAV_STATUS" == "failure" ]]; then - echo "::error::ClamAV antivirus scan detected threats. See PR comment for details." - exit 1 - fi - if [[ "$VALIDATE_RESULT" == "failure" || "$VALIDATE_RESULT" == "cancelled" ]]; then - echo "::error::One or more plugin validations failed. See PR comment for details." - exit 1 - fi - if [[ "$REPORT_RESULT" != "success" ]]; then - echo "::error::Plugin validation failed. See PR comment for details." - exit 1 - fi - echo "All plugins validated successfully." + run: python -m pluginctl gate diff --git a/.github/workflows/yank-plugin-version.yml b/.github/workflows/yank-plugin-version.yml index bac13b28..954313ae 100644 --- a/.github/workflows/yank-plugin-version.yml +++ b/.github/workflows/yank-plugin-version.yml @@ -7,6 +7,7 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + PYTHONPATH: .github/pluginctl/src on: workflow_dispatch: @@ -34,39 +35,17 @@ jobs: with: fetch-depth: 0 - - name: Load app ID from config - id: config - env: - GH_APP_ID: ${{ vars.GH_APP_ID }} - GH_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} - run: | - if [[ -n "${GH_APP_ID:-}" && -n "${GH_APP_PRIVATE_KEY:-}" ]]; then - echo "app_id=${GH_APP_ID}" >> "$GITHUB_OUTPUT" - echo "use_app=true" >> "$GITHUB_OUTPUT" - else - echo "use_app=false" >> "$GITHUB_OUTPUT" - fi - - - name: Generate GitHub App token - if: steps.config.outputs.use_app == 'true' - id: app-token - uses: actions/create-github-app-token@v3 + - name: GitHub App token + id: token + uses: ./.github/actions/gh-app-token with: - client-id: ${{ steps.config.outputs.app_id }} + app-id: ${{ vars.GH_APP_ID }} private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - - name: Log fallback to actions token - if: steps.config.outputs.use_app != 'true' - run: | - printf '## ⚠️ GitHub App token not available\n\nGH_APP_ID or GH_APP_PRIVATE_KEY not configured. Falling back to `GITHUB_TOKEN` (github-actions[bot] identity).\n' >> "$GITHUB_STEP_SUMMARY" - - - name: Make scripts executable - run: chmod +x .github/scripts/publish/*.sh - - name: Yank plugin version env: - GITHUB_TOKEN: ${{ steps.config.outputs.use_app == 'true' && steps.app-token.outputs.token || github.token }} - APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + APP_SLUG: ${{ steps.token.outputs.app-slug }} GITHUB_REPOSITORY: ${{ github.repository }} GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} @@ -74,9 +53,11 @@ jobs: YANK_VERSION: ${{ inputs.version }} YANK_ISSUE: ${{ inputs.issue_number }} SOURCE_BRANCH: ${{ github.ref_name }} + WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} + WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} run: | set -o pipefail - .github/scripts/publish/yank-version.sh 2>&1 | tee yank.log + python -m pluginctl yank 2>&1 | tee yank.log - name: Generate job summary if: always() diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 00000000..939e7a02 --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,82 @@ +# Plugin registry webhooks + +`pluginctl` can emit a signed JSON event after key lifecycle points so an +external consumer (for example a Discord bot) can react to registry activity. + +Emission is **opt-in and best-effort**: + +- Configure two repository settings: `secrets.WEBHOOK_SECRET` and + `vars.WEBHOOK_URL`. +- If either is unset, emission is a silent no-op. +- A delivery failure never fails the pipeline, it logs a `::warning::` and the + workflow continues. + +## Transport + +Each event is a single `POST` to `WEBHOOK_URL` with a compact JSON body and +these headers: + +| Header | Value | +|---|---| +| `Content-Type` | `application/json` | +| `X-PluginCtl-Event` | the event name (e.g. `plugin.published`) | +| `X-PluginCtl-Delivery` | a per-delivery UUIDv4 | +| `X-PluginCtl-Signature` | `sha256=` HMAC-SHA256 of the **raw request body** using `WEBHOOK_SECRET` | + +Verify the signature over the exact bytes received, before parsing JSON. + +## Envelope + +```json +{ + "event": "plugin.published", + "delivered_at": "2026-07-21T12:00:00Z", + "repository": "Dispatcharr/Plugins", + "actor": "octocat", + "data": { } +} +``` + +## Events + +| Event | `data` fields | +|---|---| +| `pr.validated` | `pr`, `author`, `result` (`pass`\|`fail`), `plugins[]`, `checks` (`{codeql,clamav,title}`) | +| `pr.closed_unauthorized` | `pr`, `author`, `reason` | +| `pr.quarantined` | `pr`, `author`, `infected` | +| `pr.auto_merged` | `pr`, `plugins[]` | +| `plugin.published` | `plugin`, `version`, `pr`, `actor` (one per changed plugin) | +| `plugin.yanked` | `plugin`, `version`, `issue`, `rollback_pr` | + +## Reference verifier + +The signature is HMAC-SHA256 of the raw body. Any language works; here is the +Python equivalent of what the consumer must do: + +```python +import hashlib +import hmac + +def verify(secret: str, raw_body: bytes, signature_header: str) -> bool: + """signature_header is the value of X-PluginCtl-Signature, e.g. 'sha256=abcd...'.""" + if not signature_header.startswith("sha256="): + return False + expected = hmac.new(secret.encode("utf-8"), raw_body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, signature_header[len("sha256="):]) +``` + +Node.js: + +```js +const crypto = require("crypto"); + +function verify(secret, rawBody, signatureHeader) { + if (!signatureHeader.startsWith("sha256=")) return false; + const expected = crypto.createHmac("sha256", secret).update(rawBody).digest("hex"); + const got = signatureHeader.slice("sha256=".length); + return crypto.timingSafeEqual(Buffer.from(expected), Buffer.from(got)); +} +``` + +Always compare with a constant-time function (`hmac.compare_digest` / +`crypto.timingSafeEqual`). From 1f63f8718c7e14fb311a680c24c1de839a5f3fed Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 22 Jul 2026 11:49:55 -0400 Subject: [PATCH 02/13] Run pluginctl test suite on tooling changes and gate repo-update PRs on it Adds a tests.yml workflow that runs the pluginctl pytest suite, triggered on push to main for any change outside plugins/, and also exposed via workflow_call so validate-plugin.yml can run it on a PR's merge commit. It runs with a read-only token and no secrets. - validate-plugin.yml gains a test-suite job that calls tests.yml for authorized repo-update PRs (outside_files present, not a violation, not closing). - The result flows through report (a "Tooling test suite" comment section on failure, folded into overall_failed) and through the Plugin PR Check gate, which now fails a repo-update PR with red tests even when validation is otherwise skipped. gate.evaluate and report.build_comment take a test_result; cmd_gate/cmd_report read TEST_RESULT. - Branch protection is unchanged: still one required check (Plugin PR Check). --- .github/pluginctl/src/pluginctl/cli.py | 1 + .../pluginctl/src/pluginctl/validate/gate.py | 7 ++- .../src/pluginctl/validate/report.py | 15 +++++- .github/pluginctl/tests/test_labels_gate.py | 20 ++++++++ .github/pluginctl/tests/test_report.py | 19 +++++++ .github/workflows/tests.yml | 50 +++++++++++++++++++ .github/workflows/validate-plugin.yml | 22 ++++++-- 7 files changed, 129 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/tests.yml diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py index 1e7a58e8..c62e6a95 100644 --- a/.github/pluginctl/src/pluginctl/cli.py +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -87,6 +87,7 @@ def cmd_gate(args) -> int: clamav_status=_env("CLAMAV_STATUS"), validate_result=_env("VALIDATE_RESULT"), report_result=_env("REPORT_RESULT"), + test_result=_env("TEST_RESULT", "skipped"), ) if result.ok: actions.log(result.message) diff --git a/.github/pluginctl/src/pluginctl/validate/gate.py b/.github/pluginctl/src/pluginctl/validate/gate.py index 24506bd7..d2f71ec5 100644 --- a/.github/pluginctl/src/pluginctl/validate/gate.py +++ b/.github/pluginctl/src/pluginctl/validate/gate.py @@ -20,10 +20,15 @@ def evaluate(detect_result: str, close_pr: str, skip_validation: str, outside_violation: str, title_result: str, codeql_result: str, codeql_status: str, clamav_result: str, clamav_status: str, - validate_result: str, report_result: str) -> GateResult: + validate_result: str, report_result: str, + test_result: str = "skipped") -> GateResult: """Return pass/fail with the exact ::error:: message the Bash ladder emits.""" if detect_result != "success": return GateResult(False, "Plugin detection failed or no plugin changes found.") + # Tooling test suite gates repo-update PRs. Checked before the skip_validation + # early-pass so a pure repo update cannot merge with red tests. + if test_result in ("failure", "cancelled"): + return GateResult(False, "Tooling test suite failed. See the Test Suite job for details.") if skip_validation == "true": return GateResult(True, "No plugin changes detected and author has write access - passing.") if title_result == "failure": diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 4c430600..038a98d0 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -74,7 +74,7 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, codeql_mediums: str, codeql_lows: str, codeql_unscanned_langs: str, clamav_result: str, clamav_infected: str, title_valid: str, title_feedback: str, title_suggestion: str, repository: str, - fragment_failed: bool, + fragment_failed: bool, test_result: str = "skipped", codeql_findings: str = "", codeql_medium_findings: str = "", codeql_low_findings: str = "", clamav_findings: str = "", other_plugins_section: str = "") -> str: @@ -82,6 +82,8 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, overall_failed = fragment_failed if title_valid and title_valid != "true": overall_failed = True + if test_result in ("failure", "cancelled"): + overall_failed = True L.append(MARKER) L.append("") @@ -252,6 +254,16 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, L.append(f"**Suggested format:** `{title_suggestion}`") L.append("") + if test_result in ("failure", "cancelled"): + L.append("") + L.append("---") + L.append("") + L.append("### ❌ Tooling test suite") + L.append("") + L.append("The automation test suite (`pluginctl`) failed for this change. " + "See the **Test Suite** job in this workflow run for details.") + L.append("") + L.append("") L.append("---") L.append("") @@ -386,6 +398,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, title_suggestion=os.environ.get("TITLE_SUGGESTION", ""), repository=repo, fragment_failed=parsed.any_failed, + test_result=os.environ.get("TEST_RESULT", "skipped"), codeql_findings=_read("codeql-findings/codeql-findings.md"), codeql_medium_findings=_read("codeql-medium-findings/codeql-medium-findings.md"), codeql_low_findings=_read("codeql-low-findings/codeql-low-findings.md"), diff --git a/.github/pluginctl/tests/test_labels_gate.py b/.github/pluginctl/tests/test_labels_gate.py index dcb6ba52..93c0eae9 100644 --- a/.github/pluginctl/tests/test_labels_gate.py +++ b/.github/pluginctl/tests/test_labels_gate.py @@ -48,6 +48,26 @@ def test_gate_all_success(): assert r.ok and "validated successfully" in r.message +def test_gate_test_suite_failure_fails_even_when_skip_validation(): + # A pure repo update (skip_validation=true) must not pass with red tests. + r = gate.evaluate("success", "false", "true", "false", "", "", "", "", "", "", "", + test_result="failure") + assert not r.ok and "test suite" in r.message.lower() + + +def test_gate_test_suite_skipped_allows_skip_validation_pass(): + r = gate.evaluate("success", "false", "true", "false", "", "", "", "", "", "", "", + test_result="skipped") + assert r.ok + + +def test_gate_test_suite_success_allows_pass(): + r = gate.evaluate("success", "false", "false", "false", "success", + "success", "success", "success", "success", "success", "success", + test_result="success") + assert r.ok and "validated successfully" in r.message + + def test_gate_ladder_order_outside_violation_before_close(): # outside_violation takes precedence over close_pr in the ladder r = gate.evaluate("success", "true", "false", "true", "success", diff --git a/.github/pluginctl/tests/test_report.py b/.github/pluginctl/tests/test_report.py index 8c1bd9a9..93a06a0a 100644 --- a/.github/pluginctl/tests/test_report.py +++ b/.github/pluginctl/tests/test_report.py @@ -68,6 +68,25 @@ def test_title_invalid_block(): assert "## ❌ Validation failed" in c +def test_test_suite_failure_section_and_summary(): + c = report.build_comment(**_base_kwargs(test_result="failure")) + assert "### ❌ Tooling test suite" in c + assert "pluginctl" in c + assert "## ❌ Validation failed" in c + assert "🎉" not in c + + +def test_test_suite_success_renders_nothing(): + c = report.build_comment(**_base_kwargs(test_result="success")) + assert "Tooling test suite" not in c + assert "## 🎉 All validation checks passed!" in c + + +def test_test_suite_skipped_renders_nothing(): + c = report.build_comment(**_base_kwargs(test_result="skipped")) + assert "Tooling test suite" not in c + + def test_codeql_skipped_notice_no_separator_when_only_success(): c = report.build_comment(**_base_kwargs(codeql_result="success")) # success + no unscanned -> no findings separator, still passes diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..0b480773 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,50 @@ +name: Tests + +# Runs the pluginctl pytest suite. Dual purpose: +# - on push to main for any change outside plugins/ (tooling CI) +# - via workflow_call from validate-plugin.yml, so a repo-update PR's result +# feeds the Plugin PR Check gate and the PR comment +# Runs with a read-only token and no secrets, so testing PR merge-ref code is +# low-risk. + +on: + push: + branches: [main] + paths-ignore: + - 'plugins/**' + workflow_dispatch: + workflow_call: + inputs: + ref: + description: Git ref to test (empty checks out the default for the event). + type: string + required: false + default: '' + +permissions: + contents: read + +jobs: + pytest: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + persist-credentials: false + sparse-checkout: .github/pluginctl + sparse-checkout-cone-mode: false + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install pytest + run: python -m pip install --quiet pytest + + - name: Run test suite + working-directory: .github/pluginctl + run: python -m pytest -q diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 9b19ddce..68007c15 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -203,6 +203,20 @@ jobs: pr_number: ${{ github.event.pull_request.number }} base_ref: ${{ github.event.pull_request.base.ref }} + # -------------------------------------------------------------------------- + # Job 6b: Tooling test suite (reusable workflow) for authorized repo updates + # -------------------------------------------------------------------------- + test-suite: + needs: [detect-changes] + if: >- + needs.detect-changes.result == 'success' && + needs.detect-changes.outputs.outside_files != '' && + needs.detect-changes.outputs.outside_violation != 'true' && + needs.detect-changes.outputs.close_pr == 'false' + uses: ./.github/workflows/tests.yml + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + # -------------------------------------------------------------------------- # Job 7: Validate each plugin in parallel via matrix # -------------------------------------------------------------------------- @@ -261,8 +275,8 @@ jobs: # Job 8: Aggregate all fragments, post PR comment, set final status # -------------------------------------------------------------------------- report: - needs: [detect-changes, validate-plugin, codeql-analyze, clamav-scan, validate-title] - if: always() && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.close_pr == 'false' && (needs.detect-changes.outputs.skip_validation != 'true' || needs.detect-changes.outputs.pub_key_changed == 'true' || needs.validate-title.outputs.title_valid == 'false') + needs: [detect-changes, validate-plugin, codeql-analyze, clamav-scan, validate-title, test-suite] + if: always() && needs.detect-changes.result == 'success' && needs.detect-changes.outputs.close_pr == 'false' && (needs.detect-changes.outputs.skip_validation != 'true' || needs.detect-changes.outputs.pub_key_changed == 'true' || needs.validate-title.outputs.title_valid == 'false' || needs.test-suite.result == 'failure') runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -337,6 +351,7 @@ jobs: TITLE_VALID: ${{ needs.validate-title.outputs.title_valid }} TITLE_FEEDBACK: ${{ needs.validate-title.outputs.title_feedback }} TITLE_SUGGESTION: ${{ needs.validate-title.outputs.title_suggestion }} + TEST_RESULT: ${{ needs.test-suite.result }} BASE_REF: ${{ github.event.pull_request.base.ref }} WEBHOOK_URL: ${{ vars.WEBHOOK_URL }} WEBHOOK_SECRET: ${{ secrets.WEBHOOK_SECRET }} @@ -401,7 +416,7 @@ jobs: # -------------------------------------------------------------------------- plugin-pr-check: name: Plugin PR Check - needs: [detect-changes, codeql-analyze, clamav-scan, validate-plugin, validate-title, report] + needs: [detect-changes, codeql-analyze, clamav-scan, validate-plugin, validate-title, report, test-suite] if: always() runs-on: ubuntu-latest timeout-minutes: 2 @@ -439,4 +454,5 @@ jobs: CLAMAV_STATUS: ${{ needs.clamav-scan.outputs.clamav_status }} VALIDATE_RESULT: ${{ needs.validate-plugin.result }} REPORT_RESULT: ${{ needs.report.result }} + TEST_RESULT: ${{ needs.test-suite.result }} run: python -m pluginctl gate From 29e1f148a2d32a1b786af43c49c348461ebf6dc5 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 11:55:14 -0400 Subject: [PATCH 03/13] Add regression tests for zero-padded version segments Upstream Plugins#193 fixed a Bash octal-parsing crash in version_greater_than() for versions like 0.2.08. This repo's pluginctl already avoids the bug since Python's int() has no octal ambiguity, but there was no test coverage proving it. --- .github/pluginctl/tests/test_version.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/pluginctl/tests/test_version.py b/.github/pluginctl/tests/test_version.py index ab893656..126c1dcc 100644 --- a/.github/pluginctl/tests/test_version.py +++ b/.github/pluginctl/tests/test_version.py @@ -26,6 +26,9 @@ def test_is_dispatcharr_version(s, ok): ("1.0.0", "1.0.0", False), # equal is not greater ("1.0.0", "1.0.1", False), ("0.9.9", "1.0.0", False), + ("0.2.08", "0.2.07", True), # zero-padded segment (upstream Plugins#193 regression) + ("1.09.0", "1.08.0", True), # zero-padded minor segment + ("1.0.08", "1.0.8", False), # equal numeric value across padding, not greater ]) def test_version_greater_than(new, old, gt): assert v.version_greater_than(new, old) is gt From 74bc670330ab410c202c3d6e882f535dc4b201c9 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 12:46:31 -0400 Subject: [PATCH 04/13] Simplify pluginctl: dedupe git/identity/hashing/timestamp helpers, trim dead code Consolidates duplicated git subprocess wrapping and bot-identity setup into core/git.py, adds shared core/hashing.py and core/timeutil.py helpers, drops the unused core/models.Plugin, collapses redundant git-log/API/set_output calls, and removes assorted copy-paste and dead conditionals across validate/ and publish/. Behavior-preserving; pytest suite (130 tests) still passes. --- .github/pluginctl/README.md | 2 +- .github/pluginctl/src/pluginctl/cli.py | 17 ++-- .../pluginctl/src/pluginctl/core/actions.py | 31 +++--- .github/pluginctl/src/pluginctl/core/git.py | 37 ++++--- .../pluginctl/src/pluginctl/core/hashing.py | 17 ++++ .../pluginctl/src/pluginctl/core/models.py | 97 ------------------- .../pluginctl/src/pluginctl/core/timeutil.py | 10 ++ .../pluginctl/src/pluginctl/core/version.py | 9 ++ .../src/pluginctl/integrations/automerge.py | 16 ++- .../pluginctl/integrations/external_readme.py | 14 +-- .../src/pluginctl/integrations/webhooks.py | 5 +- .../src/pluginctl/publish/cleanup.py | 12 +-- .../src/pluginctl/publish/manifest.py | 60 ++++++------ .../src/pluginctl/publish/readmes.py | 93 +++++++++--------- .../pluginctl/src/pluginctl/publish/run.py | 62 ++++++------ .../pluginctl/src/pluginctl/publish/yank.py | 19 +--- .../pluginctl/src/pluginctl/publish/zips.py | 37 +++---- .../src/pluginctl/validate/clamav.py | 8 +- .../src/pluginctl/validate/detect.py | 64 ++++++------ .../src/pluginctl/validate/plugin.py | 26 ++--- .../src/pluginctl/validate/report.py | 11 ++- .../pluginctl/src/pluginctl/validate/sarif.py | 10 +- .github/pluginctl/tests/test_integrations.py | 6 +- 23 files changed, 294 insertions(+), 369 deletions(-) create mode 100644 .github/pluginctl/src/pluginctl/core/hashing.py delete mode 100644 .github/pluginctl/src/pluginctl/core/models.py create mode 100644 .github/pluginctl/src/pluginctl/core/timeutil.py diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md index 3410b6fa..10fbf6a2 100644 --- a/.github/pluginctl/README.md +++ b/.github/pluginctl/README.md @@ -13,7 +13,7 @@ runner; there is no build step. ``` src/pluginctl/ cli.py argparse dispatch (one subcommand per verb) - core/ shared infra: actions, git, gh, jsonio, models, version + core/ shared infra: actions, git, gh, jsonio, hashing, timeutil, version validate/ PR-time pipeline: detect, title, labels, gate, sarif, langs, clamav, plugin (per-plugin checks), report publish/ releases pipeline: manifest, zips, cleanup, readmes, run, yank diff --git a/.github/pluginctl/src/pluginctl/cli.py b/.github/pluginctl/src/pluginctl/cli.py index c62e6a95..f6fb4b57 100644 --- a/.github/pluginctl/src/pluginctl/cli.py +++ b/.github/pluginctl/src/pluginctl/cli.py @@ -32,6 +32,11 @@ def _env(name: str, default: str = "") -> str: return os.environ.get(name, default) +def _repo(args) -> str: + """``--repo``, falling back to the repository the workflow runs in.""" + return args.repo or _env("GITHUB_REPOSITORY") + + # --------------------------------------------------------------------------- # # Handlers # --------------------------------------------------------------------------- # @@ -43,7 +48,7 @@ def cmd_detect(args) -> int: head_ref=args.head_ref or "", author_blacklist=_env("AUTHOR_BLACKLIST"), plugin_blacklist=_env("PLUGIN_BLACKLIST"), - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -100,7 +105,7 @@ def cmd_sarif(args) -> int: from .validate import sarif return sarif.run( results_dir=args.results_dir, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), sha=args.sha, matrix=_matrix(args.matrix), analyze_outcome=args.analyze_outcome, @@ -118,7 +123,7 @@ def cmd_validate(args) -> int: pr_author=args.author, base_ref=args.base_ref, output_file=args.out, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -130,7 +135,7 @@ def cmd_report(args) -> int: plugin_count=args.plugin_count, close_pr=_bool(args.close_pr), fragments_dir=args.fragments_dir, - repo=args.repo or _env("GITHUB_REPOSITORY"), + repo=_repo(args), ) @@ -148,8 +153,8 @@ def cmd_clamav_report(args) -> int: def cmd_webhook(args) -> int: from .integrations import webhooks data = json.loads(args.data) if args.data else {} - sent = webhooks.emit(args.event, data) - return 0 if sent or True else 1 # emission failures never fail the pipeline + webhooks.emit(args.event, data) + return 0 # emission failures never fail the pipeline def cmd_publish(args) -> int: diff --git a/.github/pluginctl/src/pluginctl/core/actions.py b/.github/pluginctl/src/pluginctl/core/actions.py index 3ba14f15..6e5d241d 100644 --- a/.github/pluginctl/src/pluginctl/core/actions.py +++ b/.github/pluginctl/src/pluginctl/core/actions.py @@ -11,28 +11,35 @@ import uuid -def set_output(name: str, value: str) -> None: - """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe). +def _format_output(name: str, value: str) -> str: + """One ``$GITHUB_OUTPUT`` entry; multiline values use GitHub's heredoc form + with a random delimiter to avoid collisions with the payload.""" + if "\n" in value: + delim = f"ghadelim_{uuid.uuid4().hex}" + return f"{name}<<{delim}\n{value}\n{delim}\n" + return f"{name}={value}\n" + - Values containing a newline use the heredoc form GitHub documents, with a - random delimiter to avoid collisions with the payload. - """ +def set_output(name: str, value: str) -> None: + """Append ``name=value`` to ``$GITHUB_OUTPUT`` (multiline-safe).""" path = os.environ.get("GITHUB_OUTPUT") if not path: # Local/dry-run: echo to stdout the same way the shell scripts did. print(f"{name}={value}") return with open(path, "a", encoding="utf-8") as fh: - if "\n" in value: - delim = f"ghadelim_{uuid.uuid4().hex}" - fh.write(f"{name}<<{delim}\n{value}\n{delim}\n") - else: - fh.write(f"{name}={value}\n") + fh.write(_format_output(name, value)) def set_outputs(**kwargs: str) -> None: - for name, value in kwargs.items(): - set_output(name, "" if value is None else str(value)) + values = {name: "" if value is None else str(value) for name, value in kwargs.items()} + path = os.environ.get("GITHUB_OUTPUT") + if not path: + for name, value in values.items(): + print(f"{name}={value}") + return + with open(path, "a", encoding="utf-8") as fh: + fh.write("".join(_format_output(name, value) for name, value in values.items())) def step_summary(markdown: str) -> None: diff --git a/.github/pluginctl/src/pluginctl/core/git.py b/.github/pluginctl/src/pluginctl/core/git.py index d347f587..9c8f233b 100644 --- a/.github/pluginctl/src/pluginctl/core/git.py +++ b/.github/pluginctl/src/pluginctl/core/git.py @@ -6,7 +6,8 @@ from typing import Optional -def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: + """``git `` with output captured as text.""" return subprocess.run( ["git", *args], check=check, @@ -15,15 +16,29 @@ def _run(args: list[str], check: bool = True) -> subprocess.CompletedProcess: ) +def configure_identity(app_slug: str) -> None: + """Set the committer identity to the GitHub App bot, else github-actions[bot].""" + if not app_slug: + run("config", "user.name", "github-actions[bot]") + run("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + return + from . import gh + bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" + run("config", "user.name", f"{app_slug}[bot]") + email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id + else f"{app_slug}[bot]@users.noreply.github.com") + run("config", "user.email", email) + + def merge_base(a: str, b: str) -> str: - return _run(["merge-base", a, b]).stdout.strip() + return run("merge-base", a, b).stdout.strip() def diff_name_only(base: str, head: str = "HEAD", paths: Optional[list[str]] = None) -> list[str]: args = ["diff", "--name-only", base, head] if paths: args += ["--", *paths] - out = _run(args).stdout + out = run(*args).stdout return [line for line in out.splitlines() if line] @@ -32,20 +47,20 @@ def diff_name_only_range(range_spec: str, paths: Optional[list[str]] = None) -> args = ["diff", "--name-only", range_spec] if paths: args += ["--", *paths] - out = _run(args).stdout + out = run(*args).stdout return [line for line in out.splitlines() if line] def show(ref_path: str) -> Optional[str]: """``git show :`` returning None when the object does not exist.""" - proc = _run(["show", ref_path], check=False) + proc = run("show", ref_path, check=False) if proc.returncode != 0: return None return proc.stdout def object_exists(ref_path: str) -> bool: - return _run(["show", ref_path], check=False).returncode == 0 + return run("show", ref_path, check=False).returncode == 0 def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: @@ -53,7 +68,7 @@ def log_format(fmt: str, ref: str, path: Optional[str] = None) -> Optional[str]: args = ["log", "-1", f"--format={fmt}", ref] if path is not None: args += ["--", path] - proc = _run(args, check=False) + proc = run(*args, check=False) if proc.returncode != 0: return None out = proc.stdout.strip() @@ -65,16 +80,16 @@ def rev_parse(rev: str, short: bool = False) -> str: if short: args.append("--short") args.append(rev) - return _run(args).stdout.strip() + return run(*args).stdout.strip() def fetch(*args: str) -> None: - _run(["fetch", *args], check=False) + run("fetch", *args, check=False) def sparse_checkout_add(path: str) -> bool: - return _run(["sparse-checkout", "add", path], check=False).returncode == 0 + return run("sparse-checkout", "add", path, check=False).returncode == 0 def checkout(*args: str) -> bool: - return _run(["checkout", *args], check=False).returncode == 0 + return run("checkout", *args, check=False).returncode == 0 diff --git a/.github/pluginctl/src/pluginctl/core/hashing.py b/.github/pluginctl/src/pluginctl/core/hashing.py new file mode 100644 index 00000000..98e6a50c --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/hashing.py @@ -0,0 +1,17 @@ +"""Chunked file hashing shared by the ZIP builder and the ClamAV report.""" + +from __future__ import annotations + +import hashlib + +CHUNK_SIZE = 1 << 20 + + +def file_digests(path: str, *algorithms: str) -> list[str]: + """Hex digests of ``path`` for each named algorithm, reading the file once.""" + hashers = [hashlib.new(name) for name in algorithms] + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(CHUNK_SIZE), b""): + for hasher in hashers: + hasher.update(chunk) + return [hasher.hexdigest() for hasher in hashers] diff --git a/.github/pluginctl/src/pluginctl/core/models.py b/.github/pluginctl/src/pluginctl/core/models.py deleted file mode 100644 index d88625b3..00000000 --- a/.github/pluginctl/src/pluginctl/core/models.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Typed view over a ``plugin.json`` document. - -Wraps the parsed dict and exposes the handful of accessors the validate/publish -logic needs, with the same ``// ""`` / ``// false`` defaulting the jq programs -used. ``raw`` stays available for the manifest builders that must preserve the -original key set and ordering. -""" - -from __future__ import annotations - -import json -from dataclasses import dataclass, field -from typing import Any, Optional - - -@dataclass -class Plugin: - raw: dict = field(default_factory=dict) - - # ---- constructors ------------------------------------------------------- - @classmethod - def from_json_text(cls, text: str) -> "Plugin": - return cls(raw=json.loads(text)) - - @classmethod - def from_file(cls, path: str) -> "Plugin": - with open(path, encoding="utf-8") as fh: - return cls(raw=json.load(fh)) - - # ---- scalar accessors (jq `// ""` semantics) ---------------------------- - def str_field(self, key: str, default: str = "") -> str: - val = self.raw.get(key) - return default if val is None else str(val) - - @property - def name(self) -> str: - return self.str_field("name") - - @property - def version(self) -> str: - return self.str_field("version") - - @property - def description(self) -> str: - return self.str_field("description") - - @property - def author(self) -> str: - return self.str_field("author") - - @property - def license(self) -> str: - return self.str_field("license") - - @property - def repo_url(self) -> str: - return self.str_field("repo_url") - - @property - def discord_thread(self) -> str: - return self.str_field("discord_thread") - - @property - def source_type(self) -> str: - return self.str_field("source_type", "local") - - @property - def source_url(self) -> str: - return self.str_field("source_url") - - @property - def min_dispatcharr_version(self) -> str: - return self.str_field("min_dispatcharr_version") - - @property - def max_dispatcharr_version(self) -> str: - return self.str_field("max_dispatcharr_version") - - @property - def deprecated(self) -> bool: - return self.raw.get("deprecated") is True - - @property - def unlisted(self) -> bool: - return self.raw.get("unlisted") is True - - @property - def maintainers(self) -> list[str]: - """``[.maintainers[]?]``: always a list, dropping non-list values.""" - val = self.raw.get("maintainers") - return list(val) if isinstance(val, list) else [] - - def maintainers_joined(self, sep: str = " ") -> str: - return sep.join(str(m) for m in self.maintainers) - - def get(self, key: str, default: Any = None) -> Any: - return self.raw.get(key, default) diff --git a/.github/pluginctl/src/pluginctl/core/timeutil.py b/.github/pluginctl/src/pluginctl/core/timeutil.py new file mode 100644 index 00000000..9ea1f1ef --- /dev/null +++ b/.github/pluginctl/src/pluginctl/core/timeutil.py @@ -0,0 +1,10 @@ +"""Shared timestamp formatting (one definition of the manifest/build ISO format).""" + +from __future__ import annotations + +import datetime + + +def now_iso() -> str: + """Current UTC time as ``YYYY-MM-DDTHH:MM:SSZ`` (the format every artifact uses).""" + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") diff --git a/.github/pluginctl/src/pluginctl/core/version.py b/.github/pluginctl/src/pluginctl/core/version.py index f279b8fc..25570275 100644 --- a/.github/pluginctl/src/pluginctl/core/version.py +++ b/.github/pluginctl/src/pluginctl/core/version.py @@ -87,3 +87,12 @@ def _version_cmp(a: str, b: str) -> int: def sort_versions_desc(versions: list[str]) -> list[str]: """Sort bare version strings newest-first, like ``sort -V -r``.""" return sorted(versions, key=cmp_to_key(_version_cmp), reverse=True) + + +def versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: + """``-`` release tags for one plugin, newest-first, minus ``-latest``.""" + prefix = f"{plugin_name}-" + latest = f"{plugin_name}-latest" + versions = [t[len(prefix):] for t in all_tags + if t.startswith(prefix) and t != latest] + return [f"{prefix}{v}" for v in sort_versions_desc(versions)] diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py index bef26116..57931066 100644 --- a/.github/pluginctl/src/pluginctl/integrations/automerge.py +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -12,18 +12,14 @@ import json import os -import subprocess from dataclasses import dataclass from ..core import actions, gh +from ..core.git import run as _git BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE") -def _git(*args: str, check: bool = False) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - @dataclass class LabelDecision: ok: bool @@ -45,12 +41,12 @@ def evaluate_labels(labels: list[str], mergeable: str) -> LabelDecision: def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: """Re-derive from the diff that the PR only updates existing plugins.""" - if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}").returncode != 0: + if _git("fetch", "origin", f"pull/{pr_number}/merge:pr-merge-{pr_number}", check=False).returncode != 0: actions.notice(f"Could not fetch merge ref for PR #{pr_number} - skipping.") return False - _git("checkout", "-q", f"pr-merge-{pr_number}") - merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD").stdout.strip() - changed = _git("diff", "--name-only", merge_base, "HEAD").stdout.splitlines() + _git("checkout", "-q", f"pr-merge-{pr_number}", check=False) + merge_base = _git("merge-base", f"origin/{base_branch}", "HEAD", check=False).stdout.strip() + changed = _git("diff", "--name-only", merge_base, "HEAD", check=False).stdout.splitlines() outside = [f for f in changed if not f.startswith("plugins/")] if outside: @@ -64,7 +60,7 @@ def reverify_is_pure_update(pr_number: str, base_branch: str = "main") -> bool: return False for slug in slugs: - if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json").returncode != 0: + if _git("show", f"origin/{base_branch}:plugins/{slug}/plugin.json", check=False).returncode != 0: actions.notice(f"Plugin '{slug}' does not exist on {base_branch} - this is a new " "plugin, not an update. Skipping.") return False diff --git a/.github/pluginctl/src/pluginctl/integrations/external_readme.py b/.github/pluginctl/src/pluginctl/integrations/external_readme.py index eb95ed98..70ee1a69 100644 --- a/.github/pluginctl/src/pluginctl/integrations/external_readme.py +++ b/.github/pluginctl/src/pluginctl/integrations/external_readme.py @@ -15,7 +15,7 @@ import subprocess from typing import Optional -from ..core import actions, gh +from ..core import actions, gh, git BRANCH = "auto/dispatcharr-plugin-readme" _SOURCE_COMMIT_RE = re.compile(r"Source commit: ([0-9a-f]+)") @@ -69,18 +69,14 @@ def run(releases_ref: str = "origin/releases") -> int: source_repo = os.environ.get("SOURCE_REPO") or os.environ.get("GITHUB_REPOSITORY", "") reviewer = os.environ.get("PR_REVIEWER", "") - show = subprocess.run(["git", "show", f"{releases_ref}:README.md"], - capture_output=True, text=True) - if show.returncode != 0: + readme = git.show(f"{releases_ref}:README.md") + if readme is None: actions.error("README.md not found on the releases branch - has the Publish Plugins workflow run yet?") return 1 - readme = show.stdout - commit_msg = subprocess.run(["git", "log", "-1", "--format=%B", releases_ref], - capture_output=True, text=True).stdout + commit_msg = git.log_format("%B", releases_ref) or "" source_commit, plugin_list = extract_metadata(commit_msg) - releases_commit = subprocess.run(["git", "rev-parse", "--short", releases_ref], - capture_output=True, text=True).stdout.strip() + releases_commit = git.rev_parse(releases_ref, short=True) contrib_url = f"https://github.com/{source_repo}/blob/main/CONTRIBUTING.md" readme = strip_preamble(readme, contrib_url) diff --git a/.github/pluginctl/src/pluginctl/integrations/webhooks.py b/.github/pluginctl/src/pluginctl/integrations/webhooks.py index 1bd71e7d..9abc4bb7 100644 --- a/.github/pluginctl/src/pluginctl/integrations/webhooks.py +++ b/.github/pluginctl/src/pluginctl/integrations/webhooks.py @@ -13,7 +13,6 @@ from __future__ import annotations -import datetime import hashlib import hmac import json @@ -24,6 +23,7 @@ from typing import Any, Optional from ..core import actions +from ..core.timeutil import now_iso USER_AGENT = "pluginctl-webhook/1" @@ -37,8 +37,7 @@ def sign(secret: str, body: bytes) -> str: def build_envelope(event: str, data: dict, repository: str, actor: str) -> dict: return { "event": event, - "delivered_at": datetime.datetime.now(datetime.timezone.utc) - .strftime("%Y-%m-%dT%H:%M:%SZ"), + "delivered_at": now_iso(), "repository": repository, "actor": actor, "data": data, diff --git a/.github/pluginctl/src/pluginctl/publish/cleanup.py b/.github/pluginctl/src/pluginctl/publish/cleanup.py index f090a242..c8587ce0 100644 --- a/.github/pluginctl/src/pluginctl/publish/cleanup.py +++ b/.github/pluginctl/src/pluginctl/publish/cleanup.py @@ -10,15 +10,7 @@ import shutil from ..core import actions, gh -from ..core.version import sort_versions_desc - - -def _versioned_tags(all_tags: list[str], plugin_name: str) -> list[str]: - prefix = f"{plugin_name}-" - latest = f"{plugin_name}-latest" - versions = [t[len(prefix):] for t in all_tags - if t.startswith(prefix) and t != latest] - return [f"{prefix}{v}" for v in sort_versions_desc(versions)] +from ..core.version import versioned_tags def run(repository: str, max_versioned_zips: int = 10) -> int: @@ -38,7 +30,7 @@ def run(repository: str, max_versioned_zips: int = 10) -> int: # Prune old versioned releases per plugin (keep the newest max_versioned_zips). for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) - tags = _versioned_tags(all_tags, plugin_name) + tags = versioned_tags(all_tags, plugin_name) if len(tags) <= max_versioned_zips: continue for old_tag in tags[max_versioned_zips:]: diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py index 5a7e8a16..8940a774 100644 --- a/.github/pluginctl/src/pluginctl/publish/manifest.py +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -19,6 +19,27 @@ from ..core import jsonio from ..core.jsonio import drop_none +from ..core.timeutil import now_iso +from ..core.version import versioned_tags + + +def _metadata_fields(metadata: dict, min_da, max_da) -> dict: + """The build-metadata fields shared by ``versions[]`` and ``latest`` entries. + + Key order matters: it is the jq emission order the Bash script produced. + """ + return { + "version": metadata.get("version"), + "commit_sha": metadata.get("commit_sha"), + "commit_sha_short": metadata.get("commit_sha_short"), + "build_timestamp": metadata.get("build_timestamp"), + "last_updated": metadata.get("last_updated"), + "checksum_md5": metadata.get("checksum_md5"), + "checksum_sha256": metadata.get("checksum_sha256"), + "min_dispatcharr_version": min_da, + "max_dispatcharr_version": max_da, + "source_url": metadata.get("source_url"), + } def build_version_entry(metadata: dict, url: str, size, canonical_version: str) -> dict: @@ -30,16 +51,9 @@ def build_version_entry(metadata: dict, url: str, size, canonical_version: str) """ if metadata: return drop_none({ - "version": metadata.get("version"), - "commit_sha": metadata.get("commit_sha"), - "commit_sha_short": metadata.get("commit_sha_short"), - "build_timestamp": metadata.get("build_timestamp"), - "last_updated": metadata.get("last_updated"), - "checksum_md5": metadata.get("checksum_md5"), - "checksum_sha256": metadata.get("checksum_sha256"), - "min_dispatcharr_version": metadata.get("min_dispatcharr_version"), - "max_dispatcharr_version": metadata.get("max_dispatcharr_version"), - "source_url": metadata.get("source_url"), + **_metadata_fields(metadata, + metadata.get("min_dispatcharr_version"), + metadata.get("max_dispatcharr_version")), "url": url, "size": size, }) @@ -72,16 +86,9 @@ def g(key): latest = None if latest_metadata: latest = drop_none({ - "version": latest_metadata.get("version"), - "commit_sha": latest_metadata.get("commit_sha"), - "commit_sha_short": latest_metadata.get("commit_sha_short"), - "build_timestamp": latest_metadata.get("build_timestamp"), - "last_updated": latest_metadata.get("last_updated"), - "checksum_md5": latest_metadata.get("checksum_md5"), - "checksum_sha256": latest_metadata.get("checksum_sha256"), - "min_dispatcharr_version": g("min_dispatcharr_version"), - "max_dispatcharr_version": g("max_dispatcharr_version"), - "source_url": latest_metadata.get("source_url"), + **_metadata_fields(latest_metadata, + g("min_dispatcharr_version"), + g("max_dispatcharr_version")), "latest_url": latest_url, "url": versioned_zips[0]["url"] if versioned_zips else None, "size": latest_size_kb, @@ -226,8 +233,8 @@ def import_gpg_key(private_key: str) -> tuple[str, bool]: Empty ``private_key`` -> ("", False) (signing disabled, not a failure). """ + from ..core import actions if not private_key: - from ..core import actions actions.log("GPG_PRIVATE_KEY not set - signatures will be skipped.") return "", False subprocess.run(["gpg", "--batch", "--import"], input=private_key, @@ -241,7 +248,6 @@ def import_gpg_key(private_key: str) -> tuple[str, bool]: if len(parts) >= 2 and "/" in parts[1]: key_id = parts[1].split("/", 1)[1] break - from ..core import actions if key_id: actions.log(f"GPG signing enabled (key: {key_id})") return key_id, False @@ -273,11 +279,10 @@ def strip_signatures() -> None: def generate(source_branch: str, releases_branch: str, repository: str, build_meta_dir: str) -> int: """Full generate-manifest.sh: per-plugin manifests, root manifest, signing.""" - import datetime import glob from ..core import actions, gh - generated_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + generated_at = now_iso() registry_url = f"https://github.com/{repository}" registry_name = repository root_url = f"https://github.com/{repository}/releases/download" @@ -326,11 +331,8 @@ def sign_if_needed(path: str, written: bool) -> None: except ValueError: existing_manifest = None - from ..core.version import sort_versions_desc prefix = f"{plugin_name}-" - raw_versions = [t[len(prefix):] for t in all_tags - if t.startswith(prefix) and t != f"{plugin_name}-latest"] - versioned_tags = [f"{prefix}{v}" for v in sort_versions_desc(raw_versions)] + tags = versioned_tags(all_tags, plugin_name) versioned_zips: list[dict] = [] latest_metadata: dict = {} @@ -338,7 +340,7 @@ def sign_if_needed(path: str, written: bool) -> None: latest_size_kb = 0 latest_size_set = False - for release_tag in versioned_tags: + for release_tag in tags: zip_version = release_tag[len(prefix):] zip_url = f"{plugin_name}-{zip_version}/{plugin_name}-{zip_version}.zip" canonical_version = _canonical(zip_version) diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py index 9885723b..eb7cae64 100644 --- a/.github/pluginctl/src/pluginctl/publish/readmes.py +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -13,9 +13,11 @@ import glob import json import os +import re from typing import Optional from ..core import git +from ..core.timeutil import now_iso def fmt_date(value: str) -> str: @@ -167,22 +169,18 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, def _badges(license_id: str, discord_thread: str, repo_url: str) -> str: - badges = "" + badges: list[str] = [] if license_id: - badges = (f"[![License: {license_id}]" - f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" - f"(https://spdx.org/licenses/{license_id}.html)") + badges.append(f"[![License: {license_id}]" + f"(https://img.shields.io/badge/License-{shields_encode(license_id)}-blue?style=flat-square)]" + f"(https://spdx.org/licenses/{license_id}.html)") if discord_thread: - if badges: - badges += " " - badges += ("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" - f"({discord_thread})") + badges.append("[![Discord](https://img.shields.io/badge/Discord-Discussion-5865F2?style=flat-square&logo=discord&logoColor=white)]" + f"({discord_thread})") if repo_url: - if badges: - badges += " " - badges += ("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" - f"({repo_url})") - return badges + badges.append("[![Repository](https://img.shields.io/badge/GitHub-Repository-181717?style=flat-square&logo=github&logoColor=white)]" + f"({repo_url})") + return " ".join(badges) def _compat_badges(min_da: str, max_da: str) -> str: @@ -210,14 +208,11 @@ def version_count_from_manifest(manifest_file: str) -> int: # --------------------------------------------------------------------------- # # Root releases README (README.md on the releases branch) # --------------------------------------------------------------------------- # -import re as _re - - def anchor_for(name: str) -> str: """GitHub-style heading anchor: lowercase, non [a-z0-9-] -> -, collapse runs.""" a = name.lower() - a = _re.sub(r"[^a-z0-9-]", "-", a) - a = _re.sub(r"-+", "-", a) + a = re.sub(r"[^a-z0-9-]", "-", a) + a = re.sub(r"-+", "-", a) return a @@ -297,12 +292,21 @@ def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: di # --------------------------------------------------------------------------- # # Orchestrators (IO): called from the releases-branch checkout directory # --------------------------------------------------------------------------- # -def _now_iso() -> str: - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def _plugin_commit(source_branch: str, plugin_dir: str) -> tuple[str, str, str]: + """(last_updated, commit_sha, commit_sha_short) of the plugin's last commit. + + One ``git log`` call for all three fields; empty strings when there is none. + """ + out = git.log_format("%cI%n%H%n%h", f"origin/{source_branch}", plugin_dir) + if not out: + return "", "", "" + fields = out.splitlines() + fields += [""] * (3 - len(fields)) + return fields[0], fields[1], fields[2] def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: - return git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now_iso() + return _plugin_commit(source_branch, plugin_dir)[0] or now_iso() def generate_plugin_readmes(source_branch: str, repository: str) -> None: @@ -335,13 +339,16 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor root = _load_json("manifest.json") or {} root_url = (root.get("manifest") or {}).get("root_url") or "" - plugin_dirs = sorted(glob.glob("plugins/*/")) raws: dict[str, dict] = {} - for pd in plugin_dirs: + has_deprecated = False + for pd in sorted(glob.glob("plugins/*/")): pname = os.path.basename(pd.rstrip("/")) raw = _load_json(os.path.join(pd, "plugin.json")) - if raw is not None: - raws[pname] = raw + if raw is None: + continue + raws[pname] = raw + if raw.get("deprecated") is True: + has_deprecated = True L: list[str] = [] L.append("# Plugin Releases") @@ -358,20 +365,13 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor L.append("| Plugin | Version | Author | License | Description |") L.append("|--------|---------|-------|---------|-------------|") - for pass_name in ("active", "deprecated"): - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None: - continue - deprecated = raw.get("deprecated") is True + for deprecated_pass in (False, True): + for raw in raws.values(): if raw.get("unlisted") is True: continue - if pass_name == "active" and deprecated: - continue - if pass_name == "deprecated" and not deprecated: + if (raw.get("deprecated") is True) is not deprecated_pass: continue - L.append(table_row(raw, pass_name == "deprecated")) + L.append(table_row(raw, deprecated_pass)) L.append("") L.append("---") @@ -380,9 +380,10 @@ def generate_releases_readme(source_branch: str, releases_branch: str, repositor def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: pd = f"plugins/{pname}/" manifest = _load_json(f"metadata/{pname}/manifest.json") - last_updated = _plugin_last_updated(source_branch, pd) - commit_sha = git.log_format("%H", f"origin/{source_branch}", pd) or "unknown" - commit_short = git.log_format("%h", f"origin/{source_branch}", pd) or "unknown" + last_updated, commit_sha, commit_short = _plugin_commit(source_branch, pd) + last_updated = last_updated or now_iso() + commit_sha = commit_sha or "unknown" + commit_short = commit_short or "unknown" version_count = version_count_from_manifest(f"metadata/{pname}/manifest.json") has_source_readme = os.path.isfile(f"plugins/{pname}/README.md") L.append(render_plugin_block( @@ -394,25 +395,19 @@ def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: has_source_readme=has_source_readme)) # Active detailed sections - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None or raw.get("deprecated") is True or raw.get("unlisted") is True: + for pname, raw in raws.items(): + if raw.get("deprecated") is True or raw.get("unlisted") is True: continue emit_block(pname, raw, False) - has_deprecated = any( - r.get("deprecated") is True for r in raws.values()) if has_deprecated: L.append("") L.append("## Deprecated Plugins") L.append("") L.append("These plugins are deprecated and may be removed in the future.") L.append("") - for pd in plugin_dirs: - pname = os.path.basename(pd.rstrip("/")) - raw = raws.get(pname) - if raw is None or raw.get("deprecated") is not True or raw.get("unlisted") is True: + for pname, raw in raws.items(): + if raw.get("deprecated") is not True or raw.get("unlisted") is True: continue emit_block(pname, raw, True) diff --git a/.github/pluginctl/src/pluginctl/publish/run.py b/.github/pluginctl/src/pluginctl/publish/run.py index ea002d11..f06498e0 100644 --- a/.github/pluginctl/src/pluginctl/publish/run.py +++ b/.github/pluginctl/src/pluginctl/publish/run.py @@ -12,7 +12,8 @@ import subprocess import tempfile -from ..core import actions, git as _git_mod +from ..core import actions +from ..core.git import configure_identity, run as _git from . import cleanup, manifest, readmes, zips RELEASES_BRANCH = "releases" @@ -20,26 +21,6 @@ RELEASES_BRANCH_VERSION = "3" -def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - -def _configure_identity(repository: str, app_slug: str) -> None: - if app_slug: - from ..core import gh - bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" - _git("config", "user.name", f"{app_slug}[bot]") - if bot_user_id: - _git("config", "user.email", - f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com") - else: - _git("config", "user.email", f"{app_slug}[bot]@users.noreply.github.com") - else: - _git("config", "user.name", "github-actions[bot]") - _git("config", "user.email", - "41898282+github-actions[bot]@users.noreply.github.com") - - def run(source_branch: str) -> int: repository = os.environ["GITHUB_REPOSITORY"] token = os.environ["GITHUB_TOKEN"] @@ -56,7 +37,7 @@ def run(source_branch: str) -> int: try: _git("clone", "--no-checkout", remote, f"{workdir}/repo") os.chdir(f"{workdir}/repo") - _configure_identity(repository, app_slug) + configure_identity(app_slug) _setup_releases_branch(repository, source_branch, token, remote, force_rebuild, force_plugin) @@ -104,8 +85,7 @@ def _setup_releases_branch(repository, source_branch, token, remote, if force_rebuild and force_plugin: if branch_exists: - _git("checkout", RELEASES_BRANCH) - _git("pull", "origin", RELEASES_BRANCH, check=False) + _checkout_existing() else: _orphan_init() actions.log(f"Targeted force rebuild: deleting GitHub Releases for {force_plugin}") @@ -127,21 +107,23 @@ def _setup_releases_branch(repository, source_branch, token, remote, if tag.startswith(f"{pname}-"): gh.release_delete(tag, repository) subprocess.run(["rm", "-rf", "plugins"], check=False) - _git("checkout", "--orphan", RELEASES_BRANCH) - _git("rm", "-rf", ".", check=False) - _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch (force rebuild)") + _orphan_init(f"Initialize {RELEASES_BRANCH} branch (force rebuild)") _git("push", "--force", remote, RELEASES_BRANCH) elif branch_exists: - _git("checkout", RELEASES_BRANCH) - _git("pull", "origin", RELEASES_BRANCH, check=False) + _checkout_existing() else: _orphan_init() -def _orphan_init() -> None: +def _orphan_init(message: str = f"Initialize {RELEASES_BRANCH} branch") -> None: _git("checkout", "--orphan", RELEASES_BRANCH) _git("rm", "-rf", ".", check=False) - _git("commit", "--allow-empty", "-m", f"Initialize {RELEASES_BRANCH} branch") + _git("commit", "--allow-empty", "-m", message) + + +def _checkout_existing() -> None: + _git("checkout", RELEASES_BRANCH) + _git("pull", "origin", RELEASES_BRANCH, check=False) def _commit_and_push(repository, source_branch, remote) -> int: @@ -174,15 +156,27 @@ def _commit_and_push(repository, source_branch, remote) -> int: def _emit_published(repository: str, changed: str) -> None: - """Emit one plugin.published event per changed plugin (best-effort).""" + """Emit one plugin.published event per changed plugin (best-effort, concurrent). + + Deliveries are independent and already non-fatal, so they run in parallel + rather than serializing one 15s timeout per plugin on the publish path. + """ + from concurrent.futures import ThreadPoolExecutor + from ..integrations import webhooks actor = os.environ.get("GITHUB_ACTOR", "") + payloads = [] for line in changed.splitlines(): if "@" not in line: continue plugin_key, _, version = line.partition("@") - webhooks.emit("plugin.published", - webhooks.plugin_published(plugin_key.replace("_", "-"), version, None, actor)) + payloads.append(webhooks.plugin_published(plugin_key.replace("_", "-"), + version, None, actor)) + if not payloads: + return + with ThreadPoolExecutor(max_workers=min(8, len(payloads))) as pool: + for payload in payloads: + pool.submit(webhooks.emit, "plugin.published", payload) def _only_timestamps() -> bool: diff --git a/.github/pluginctl/src/pluginctl/publish/yank.py b/.github/pluginctl/src/pluginctl/publish/yank.py index fdc8963e..e001f1a9 100644 --- a/.github/pluginctl/src/pluginctl/publish/yank.py +++ b/.github/pluginctl/src/pluginctl/publish/yank.py @@ -16,16 +16,13 @@ import tempfile from ..core import actions, gh +from ..core.git import configure_identity, run as _git from ..core.version import sort_versions_desc from . import manifest, readmes RELEASES_BRANCH = "releases" -def _git(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return subprocess.run(["git", *args], check=check, capture_output=True, text=True) - - def run() -> int: repository = os.environ["GITHUB_REPOSITORY"] token = os.environ["GITHUB_TOKEN"] @@ -54,7 +51,7 @@ def run() -> int: try: _git("clone", "--no-checkout", remote, f"{workdir}/repo") os.chdir(f"{workdir}/repo") - _configure_identity(repository, app_slug) + configure_identity(app_slug) if _git("ls-remote", "--exit-code", "--heads", "origin", RELEASES_BRANCH, check=False).returncode != 0: @@ -146,18 +143,6 @@ def run() -> int: subprocess.run(["rm", "-rf", workdir, build_meta_dir], check=False) -def _configure_identity(repository: str, app_slug: str) -> None: - if app_slug: - bot_user_id = gh.api(f"/users/{app_slug}%5Bbot%5D", jq=".id") or "" - _git("config", "user.name", f"{app_slug}[bot]") - email = (f"{bot_user_id}+{app_slug}[bot]@users.noreply.github.com" if bot_user_id - else f"{app_slug}[bot]@users.noreply.github.com") - _git("config", "user.email", email) - else: - _git("config", "user.name", "github-actions[bot]") - _git("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") - - def _commit_releases(remote, plugin, version, issue) -> str: actions.log("\n=== Committing ===") subprocess.run(["rm", "-rf", "plugins"], check=False) diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py index 584ba734..6b2df0e8 100644 --- a/.github/pluginctl/src/pluginctl/publish/zips.py +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -10,7 +10,6 @@ from __future__ import annotations import glob -import hashlib import json import os import shutil @@ -21,22 +20,9 @@ from typing import Optional from ..core import actions, gh, git, jsonio +from ..core.hashing import file_digests from ..core.jsonio import drop_none - - -def _now() -> str: - import datetime - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - -def _checksums(path: str) -> tuple[str, str]: - md5 = hashlib.md5() - sha = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - md5.update(chunk) - sha.update(chunk) - return md5.hexdigest(), sha.hexdigest() +from ..core.timeutil import now_iso def _download(url: str, dest: str, attempts: int = 3) -> bool: @@ -54,6 +40,16 @@ def _download(url: str, dest: str, attempts: int = 3) -> bool: return False +def _commit_info(source_branch: str, plugin_dir: str) -> tuple[str, str, str]: + """(commit_sha, commit_sha_short, commit_date) of the last commit touching the plugin.""" + out = git.log_format("%H%n%h%n%cI", f"origin/{source_branch}", plugin_dir) + if not out: + return "", "", "" + fields = out.splitlines() + fields += [""] * (3 - len(fields)) + return fields[0], fields[1], fields[2] + + def run(source_branch: str, repository: str, build_meta_dir: str) -> int: changed: list[str] = [] open("changed_plugins.txt", "w", encoding="utf-8").close() @@ -74,10 +70,9 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: continue source_type = raw.get("source_type") or "local" - build_timestamp = _now() - commit_sha = git.log_format("%H", f"origin/{source_branch}", plugin_dir) or "" - commit_sha_short = git.log_format("%h", f"origin/{source_branch}", plugin_dir) or "" - last_updated = git.log_format("%cI", f"origin/{source_branch}", plugin_dir) or _now() + build_timestamp = now_iso() + commit_sha, commit_sha_short, commit_date = _commit_info(source_branch, plugin_dir) + last_updated = commit_date or now_iso() source_url_resolved = "" if source_type == "external": @@ -95,7 +90,7 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: subprocess.run(["zip", "-r", zip_path, plugin_key], cwd=tmpdir, check=True, stdout=subprocess.DEVNULL) - checksum_md5, checksum_sha256 = _checksums(zip_path) + checksum_md5, checksum_sha256 = file_digests(zip_path, "md5", "sha256") min_da = str(raw.get("min_dispatcharr_version") or "") max_da = str(raw.get("max_dispatcharr_version") or "") zip_size_kb = os.path.getsize(zip_path) // 1024 diff --git a/.github/pluginctl/src/pluginctl/validate/clamav.py b/.github/pluginctl/src/pluginctl/validate/clamav.py index 880d439d..201d5b20 100644 --- a/.github/pluginctl/src/pluginctl/validate/clamav.py +++ b/.github/pluginctl/src/pluginctl/validate/clamav.py @@ -6,20 +6,16 @@ from __future__ import annotations -import hashlib import os from typing import Callable, Optional from ..core import actions +from ..core.hashing import file_digests def _sha256_file(path: str) -> str: try: - h = hashlib.sha256() - with open(path, "rb") as fh: - for chunk in iter(lambda: fh.read(1 << 20), b""): - h.update(chunk) - return h.hexdigest() + return file_digests(path, "sha256")[0] except OSError: return "" diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index 63c50c4d..69f23ce7 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -23,6 +23,13 @@ def is_safe_name(name: str) -> bool: return bool(SAFE_NAME_RE.match(name)) +def author_in_plugin_json(pr_author: str, data: dict) -> bool: + """True when pr_author is the plugin's ``author`` or one of its ``maintainers``.""" + author = data.get("author") or "" + maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] + return pr_author == author or pr_author in maintainers + + def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) -> bool: """True when pr_author is the base-branch author or in base maintainers. @@ -34,9 +41,7 @@ def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) data = json.loads(base_json_text) except json.JSONDecodeError: return False - author = data.get("author") or "" - maintainers = [str(m) for m in (data.get("maintainers") or []) if m is not None] - return pr_author == author or pr_author in maintainers + return author_in_plugin_json(pr_author, data) @dataclass @@ -50,37 +55,37 @@ def check_blacklists(pr_author: str, plugins: list[str], """Case-insensitive author/plugin blacklist check (whitespace-trimmed entries).""" author_bl = (author_blacklist or "").strip() plugin_bl = (plugin_blacklist or "").strip() - if not author_bl and not plugin_bl: - return BlacklistResult() - if author_bl: - for entry in author_bl.split(","): - slug = entry.replace(" ", "") - if pr_author.lower() == slug.lower(): - actions.warning(f"PR author '{pr_author}' is on the author blacklist.") - return BlacklistResult(True, "author-blacklisted") + if author_bl and _csv_contains(pr_author, author_bl): + actions.warning(f"PR author '{pr_author}' is on the author blacklist.") + return BlacklistResult(True, "author-blacklisted") if plugin_bl: - blocked = plugin_bl.split(",") for plugin in plugins: - if not plugin: - continue - for entry in blocked: - slug = entry.replace(" ", "") - if plugin.lower() == slug.lower(): - actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") - return BlacklistResult(True, "plugin-blacklisted") + if plugin and _csv_contains(plugin, plugin_bl): + actions.warning(f"Plugin '{plugin}' is on the plugin blacklist.") + return BlacklistResult(True, "plugin-blacklisted") return BlacklistResult() +def _csv_contains(value: str, csv: str) -> bool: + """Case-insensitive membership in a comma-separated list (spaces stripped).""" + return any(value.lower() == entry.replace(" ", "").lower() for entry in csv.split(",")) + + def run(pr_author: str, base_ref: str, head_ref: str = "", author_blacklist: str = "", plugin_blacklist: str = "", repo: str = "") -> int: """Execute detection; write outputs; return process exit code (0 ok, 1 hard block).""" owner, _, name = repo.partition("/") + cached_write_access: Optional[bool] = None def write_access() -> bool: - return gh.has_write_access(owner, name, pr_author) + """``gh api`` permission lookup, resolved at most once per run.""" + nonlocal cached_write_access + if cached_write_access is None: + cached_write_access = gh.has_write_access(owner, name, pr_author) + return cached_write_access # Bot-authored yank rollback PRs bypass plugin validation entirely. if pr_author.endswith("[bot]") and head_ref.startswith("yank/"): @@ -147,20 +152,19 @@ def write_access() -> bool: close_pr = True close_reason = bl.reason - matrix_json = json.dumps(plugin_list, separators=(",", ":")) - actions.set_output("matrix", matrix_json) - actions.set_output("plugin_count", str(plugin_count)) - actions.set_output("close_pr", "true" if close_pr else "false") - actions.set_output("skip_validation", "false") - actions.set_output("outside_violation", "true" if has_outside_violation else "false") - actions.set_output("close_reason", close_reason) + actions.set_outputs(matrix=json.dumps(plugin_list, separators=(",", ":")), + plugin_count=str(plugin_count), + close_pr="true" if close_pr else "false", + skip_validation="false", + outside_violation="true" if has_outside_violation else "false", + close_reason=close_reason) if outside_changes: actions.set_output("outside_files", "\n".join(outside_changes)) pub_key_changed = is_maintainer and PUB_KEY_PATH in outside_changes - actions.set_output("pub_key_changed", "true" if pub_key_changed else "false") - actions.set_output("has_new_plugin", "true" if has_new_plugin else "false") - actions.set_output("has_updated_plugin", "true" if has_updated_plugin else "false") + actions.set_outputs(pub_key_changed="true" if pub_key_changed else "false", + has_new_plugin="true" if has_new_plugin else "false", + has_updated_plugin="true" if has_updated_plugin else "false") actions.log(f"Detected {plugin_count} plugin(s): {' '.join(plugin_list)}") actions.log(f"close_pr={'true' if close_pr else 'false'}") diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py index 965951da..bfd32de4 100644 --- a/.github/pluginctl/src/pluginctl/validate/plugin.py +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -19,6 +19,7 @@ from ..core import actions, gh, git from ..core.version import is_semver, is_dispatcharr_version, version_greater_than +from .detect import author_in_plugin_json, is_safe_name METADATA_ONLY_FIELDS = [ "description", "repo_url", "discord_thread", "min_dispatcharr_version", @@ -26,7 +27,6 @@ ] _GH_DOWNLOAD_RE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/") -_GH_DOWNLOAD_TAG_RE = re.compile(r"^https://github\.com/[^/]+/[^/]+/releases/download/([^/]+)/") SPDX_URL = "https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json" @@ -137,7 +137,7 @@ def emit_fragment() -> None: _write(output_file, text) # Folder name format - if not re.match(r"^[a-z0-9]+(-[a-z0-9]+)*$", plugin_name): + if not is_safe_name(plugin_name): rows.append(f"| Folder name | ❌ | Must be lowercase-kebab-case - got " f"`{plugin_name}`, e.g. `my-plugin-name` |") failed = True @@ -221,9 +221,9 @@ def emit_fragment() -> None: release_link = f"https://github.com/{gh_owner}/{gh_repo}/releases/tag/{gh_tag}" if old_version and old_source_url_tmpl: old_resolved = old_source_url_tmpl.replace("{version}", old_version) - om = _GH_DOWNLOAD_TAG_RE.match(old_resolved) + om = _GH_DOWNLOAD_RE.match(old_resolved) if om: - old_gh_tag = om.group(1) + old_gh_tag = om.group(3) compare_link = (f"https://github.com/{gh_owner}/{gh_repo}" f"/compare/{old_gh_tag}...{gh_tag}") else: @@ -275,9 +275,7 @@ def emit_fragment() -> None: rows.append("| Permission | ✅ | You have permission to modify this plugin |") has_permission = True elif base_raw is not None: - base_author = jq_r(base_raw.get("author"), "") - base_maintainers = [str(m) for m in (base_raw.get("maintainers") or []) if m is not None] - if pr_author == base_author or pr_author in base_maintainers: + if author_in_plugin_json(pr_author, base_raw): rows.append("| Permission | ✅ | You have permission to modify this plugin |") has_permission = True else: @@ -325,15 +323,17 @@ def emit_fragment() -> None: # Dispatcharr version constraints min_da = jq_r(raw.get("min_dispatcharr_version"), "") max_da = jq_r(raw.get("max_dispatcharr_version"), "") - if min_da and not is_dispatcharr_version(min_da): + min_ok = is_dispatcharr_version(min_da) + max_ok = is_dispatcharr_version(max_da) + if min_da and not min_ok: rows.append(f"| `min_dispatcharr_version` | ❌ | `{min_da}` is not valid semver - " "expected `X.Y.Z` or `vX.Y.Z` |") failed = True - if max_da and not is_dispatcharr_version(max_da): + if max_da and not max_ok: rows.append(f"| `max_dispatcharr_version` | ❌ | `{max_da}` is not valid semver - " "expected `X.Y.Z` or `vX.Y.Z` |") failed = True - if min_da and max_da and is_dispatcharr_version(max_da) and is_dispatcharr_version(min_da): + if min_da and max_da and max_ok and min_ok: _max = max_da[1:] if max_da.startswith("v") else max_da _min = min_da[1:] if min_da.startswith("v") else min_da if not version_greater_than(_max, _min) and _max != _min: @@ -364,10 +364,10 @@ def emit_fragment() -> None: jq_r(raw.get("name"), ""), jq_r(raw.get("version"), ""), jq_r(raw.get("description"), ""), - jq_r(raw.get("author"), ""), + author, ", ".join(maintainers), - jq_r(raw.get("repo_url"), ""), - jq_r(raw.get("discord_thread"), ""), + repo_url, + discord_thread, ] meta_row = f"" diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 038a98d0..f72a187f 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -18,6 +18,13 @@ MARKER = "" +# close_reason values for which the PR itself is closed (vs. merely reported on). +CLOSING_REASONS = ("unauthorized", "author-blacklisted", "plugin-blacklisted") + + +def closes_pr(close_pr: bool, close_reason: str) -> bool: + return close_pr and close_reason in CLOSING_REASONS + @dataclass class ParsedFragments: @@ -414,7 +421,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, _emit_webhook(pr_number, pr_author, close_pr, close_reason, parsed) - if close_pr and close_reason in ("unauthorized", "author-blacklisted", "plugin-blacklisted"): + if closes_pr(close_pr, close_reason): gh.pr_close(pr_number) actions.log(f"PR #{pr_number} closed: unauthorized") return comment_exit @@ -422,7 +429,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, def _emit_webhook(pr_number, pr_author, close_pr, close_reason, parsed) -> None: from ..integrations import webhooks - if close_pr and close_reason in ("unauthorized", "author-blacklisted", "plugin-blacklisted"): + if closes_pr(close_pr, close_reason): webhooks.emit("pr.closed_unauthorized", webhooks.pr_closed_unauthorized(pr_number, pr_author, close_reason)) return diff --git a/.github/pluginctl/src/pluginctl/validate/sarif.py b/.github/pluginctl/src/pluginctl/validate/sarif.py index 04c7fada..6c9b7ff2 100644 --- a/.github/pluginctl/src/pluginctl/validate/sarif.py +++ b/.github/pluginctl/src/pluginctl/validate/sarif.py @@ -102,9 +102,9 @@ def classify(sarif_objs: list[dict]) -> Counts: for result in (run.get("results") or []): counts.total += 1 sev = severity_of(result, secmap) - if sev >= 7.0: + if is_blocking(sev): counts.blocking += 1 - elif 6.0 <= sev < 7.0: + elif is_medium(sev): counts.medium += 1 else: counts.low += 1 @@ -223,8 +223,8 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], actions.set_output("codeql_mediums", str(counts.medium)) actions.set_output("codeql_lows", str(counts.low)) - if counts.blocking > 0: - _write(results_dir and "codeql-findings.md", + if counts.blocking > 0 and results_dir: + _write("codeql-findings.md", findings_table(objs, is_blocking, repo, sha, external_prefixes)) if counts.medium > 0: _write("codeql-medium-findings.md", @@ -253,7 +253,5 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], def _write(path: str, content: str) -> None: - if not path: - return with open(path, "w", encoding="utf-8") as fh: fh.write(content) diff --git a/.github/pluginctl/tests/test_integrations.py b/.github/pluginctl/tests/test_integrations.py index 05455a01..5df9d0ce 100644 --- a/.github/pluginctl/tests/test_integrations.py +++ b/.github/pluginctl/tests/test_integrations.py @@ -1,5 +1,5 @@ +from pluginctl.core import version from pluginctl.integrations import automerge, external_readme -from pluginctl.publish import cleanup # ---- automerge label gate ---- @@ -54,7 +54,7 @@ def test_rewrite_links(): assert "[anchor](#section)" in out # anchors untouched -# ---- cleanup tag selection ---- +# ---- release tag selection ---- def test_versioned_tags_sorted_desc_excludes_latest(): tags = ["demo-1.0.0", "demo-1.10.0", "demo-1.2.0", "demo-latest", "other-1.0.0"] - assert cleanup._versioned_tags(tags, "demo") == ["demo-1.10.0", "demo-1.2.0", "demo-1.0.0"] + assert version.versioned_tags(tags, "demo") == ["demo-1.10.0", "demo-1.2.0", "demo-1.0.0"] From cf70344e4122b6cc8779f661a1eb1c817db44680 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 25 Jul 2026 13:02:21 -0400 Subject: [PATCH 05/13] Close two PR-validation trust-boundary gaps found in security review An unsafe-named plugins/ folder was silently dropped from the scan matrix instead of blocking the PR, letting it ride along unscanned to merge and publish alongside a legitimate change. detect.py now closes the PR (unsafe-plugin-name) instead, and zips.py adds the same allowlist check as defense in depth before publishing. Separately, a plugin.json description containing an embedded newline could smuggle in a forged line ahead of the genuine one, letting report.parse_fragments render attacker-chosen repo/discord links into the bot's PR comment. parse_fragments now anchors on the last marker line, and plugin.py collapses description to a single line before embedding it. --- .../pluginctl/src/pluginctl/publish/zips.py | 4 ++ .../src/pluginctl/validate/detect.py | 17 ++++++++- .../src/pluginctl/validate/plugin.py | 12 +++++- .../src/pluginctl/validate/report.py | 37 +++++++++++-------- .github/pluginctl/tests/test_detect.py | 37 +++++++++++++++++++ .github/pluginctl/tests/test_report.py | 20 ++++++++++ .github/pluginctl/tests/test_validate.py | 23 ++++++++++++ .../pluginctl/tests/test_zips_safe_name.py | 31 ++++++++++++++++ 8 files changed, 163 insertions(+), 18 deletions(-) create mode 100644 .github/pluginctl/tests/test_zips_safe_name.py diff --git a/.github/pluginctl/src/pluginctl/publish/zips.py b/.github/pluginctl/src/pluginctl/publish/zips.py index 6b2df0e8..8ce914a9 100644 --- a/.github/pluginctl/src/pluginctl/publish/zips.py +++ b/.github/pluginctl/src/pluginctl/publish/zips.py @@ -23,6 +23,7 @@ from ..core.hashing import file_digests from ..core.jsonio import drop_none from ..core.timeutil import now_iso +from ..validate.detect import is_safe_name def _download(url: str, dest: str, attempts: int = 3) -> bool: @@ -56,6 +57,9 @@ def run(source_branch: str, repository: str, build_meta_dir: str) -> int: for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) + if not is_safe_name(plugin_name): + actions.warning(f"Skipping publish for unsafe plugin folder name: '{plugin_name}'") + continue plugin_key = plugin_name.replace("-", "_") with open(os.path.join(plugin_dir, "plugin.json"), encoding="utf-8") as fh: raw = json.load(fh) diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index 69f23ce7..9572b7f3 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -107,15 +107,28 @@ def write_access() -> bool: if not plugin_list: return _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) - # Allowlist: only safe kebab-case names enter the matrix. + # Allowlist: only safe kebab-case names enter the matrix. A folder with an + # unsafe name is never scanned (ClamAV/CodeQL/publish all iterate the safe + # matrix), so its presence must block the whole PR rather than being + # silently dropped - otherwise it can ride along with a legitimate, + # passing plugin change straight to `main` and publish unscanned. safe_list: list[str] = [] + unsafe_list: list[str] = [] for plugin in plugin_list: if is_safe_name(plugin): safe_list.append(plugin) else: - actions.warning(f"Skipping plugin with unsafe folder name: '{plugin}'") + unsafe_list.append(plugin) + actions.warning(f"Unsafe plugin folder name: '{plugin}'") plugin_list = safe_list + if unsafe_list: + actions.error(f"Unsafe plugin folder name(s) detected: {', '.join(unsafe_list)}") + actions.set_outputs(close_pr="true", close_reason="unsafe-plugin-name", + plugin_count="0", matrix="[]", + has_new_plugin="false", has_updated_plugin="false") + return 0 + if not plugin_list: actions.error("No valid plugin changes detected in this PR.") actions.set_outputs(close_pr="true", close_reason="no-valid-plugins", diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py index bfd32de4..328716a7 100644 --- a/.github/pluginctl/src/pluginctl/validate/plugin.py +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -50,6 +50,12 @@ def field_present(raw: dict, key: str) -> bool: return not (val is None or val is False) +def _single_line(s: str) -> str: + """Collapse newlines/carriage returns so a free-text field can't inject + extra fragment lines (e.g. a forged ```` marker).""" + return s.replace("\r\n", " ").replace("\n", " ").replace("\r", " ") + + def tsv(fields: list[str]) -> str: """Reproduce ``@tsv`` escaping for a single row.""" def esc(s: str) -> str: @@ -115,7 +121,11 @@ def run(plugin_name: str, pr_author: str, base_ref: str, output_file: str, except (OSError, json.JSONDecodeError): raw = None if raw is not None: - desc = jq_r(raw.get("description"), "") + # Free-text fields are attacker-controlled. Collapse newlines before + # embedding them in the fragment so a value can never introduce a + # forged "" marker line ahead of the real one + # that `report.parse_fragments` looks for. + desc = _single_line(jq_r(raw.get("description"), "")) repo_url = jq_r(raw.get("repo_url"), "") if desc: pre_lines.append(f"_{desc}_") diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index f72a187f..83490ae8 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -48,23 +48,30 @@ def parse_fragments(fragments_dir: str) -> ParsedFragments: content = _read(fragment) if "❌" in content: result.any_failed = True - # META_ROW extraction - for line in content.splitlines(): + # META_ROW extraction. plugin.py always appends the genuine marker as + # the fragment's last line, so anchor here rather than taking the + # first match - a free-text field (e.g. description) earlier in the + # fragment must never be able to forge this by embedding its own + # ""): - meta = meta[:-3] - fields = meta.split("\t") - fields += [""] * (7 - len(fields)) - f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] - if f_repo or f_discord: - result.plugin_links += f"**`{f_name}`:**\n" - if f_repo: - result.plugin_links += f"- [GitHub Repository]({f_repo})\n" - if f_discord: - result.plugin_links += f"- [Discord Thread]({f_discord})\n" - result.plugin_links += "\n" + meta_line = line break + if meta_line is not None: + meta = meta_line[len(""): + meta = meta[:-3] + fields = meta.split("\t") + fields += [""] * (7 - len(fields)) + f_name, f_version, f_desc, f_author, f_maint, f_repo, f_discord = fields[:7] + if f_repo or f_discord: + result.plugin_links += f"**`{f_name}`:**\n" + if f_repo: + result.plugin_links += f"- [GitHub Repository]({f_repo})\n" + if f_discord: + result.plugin_links += f"- [Discord Thread]({f_discord})\n" + result.plugin_links += "\n" # visible = fragment minus META_ROW lines, trailing whitespace stripped ($() semantics) visible_lines = [ln for ln in content.splitlines() if not ln.startswith(""-prefixed line must not be able to spoof the + Plugin Contact Links - only the genuine, always-last marker counts.""" + f = tmp_path / "demo.fragment.md" + f.write_text( + "### Plugin: `demo`\n\n" + "_Cool plugin_\n" + "\n" + "\n| ok |\n" + "\n", + encoding="utf-8", + ) + parsed = report.parse_fragments(str(tmp_path)) + assert "evil.example" not in parsed.plugin_links + assert "discord.gg/evil" not in parsed.plugin_links + assert "**`Demo`:**" in parsed.plugin_links + assert "- [GitHub Repository](https://real.example)" in parsed.plugin_links assert parsed.any_failed is False diff --git a/.github/pluginctl/tests/test_validate.py b/.github/pluginctl/tests/test_validate.py index 1ae6ec41..6a0bba73 100644 --- a/.github/pluginctl/tests/test_validate.py +++ b/.github/pluginctl/tests/test_validate.py @@ -107,3 +107,26 @@ def test_unauthorized_author_new_plugin(plugin_repo): text = frag.read_text(encoding="utf-8") assert '| Permission | ❌ | Add `"author": "mallory"` to plugin.json |' in text assert _outputs(out)["has_permission"] == "false" + + +def test_description_newline_cannot_forge_meta_row(plugin_repo): + """A description embedding its own ""-prefixed line + must not produce that line in the fragment - only the genuine, always-last + marker plugin.py itself appends should ever start with that prefix.""" + tmp_path, out = plugin_repo + pj = tmp_path / "plugins" / "demo-plugin" / "plugin.json" + data = json.loads(pj.read_text()) + data["description"] = ("Cool plugin\n") + pj.write_text(json.dumps(data), encoding="utf-8") + frag = tmp_path / "frag.md" + validate.run("demo-plugin", "alice", "main", str(frag), repo="org/repo") + lines = frag.read_text(encoding="utf-8").splitlines() + meta_lines = [ln for ln in lines if ln.startswith("")].split("\t") + f_repo, f_discord = fields[5], fields[6] + assert "evil.example" not in f_repo + assert "discord.gg/evil" not in f_discord diff --git a/.github/pluginctl/tests/test_zips_safe_name.py b/.github/pluginctl/tests/test_zips_safe_name.py new file mode 100644 index 00000000..fc55ea93 --- /dev/null +++ b/.github/pluginctl/tests/test_zips_safe_name.py @@ -0,0 +1,31 @@ +import json + +from pluginctl.core import gh +from pluginctl.publish import zips + + +def test_run_skips_unsafe_named_plugin_directory(tmp_path, monkeypatch): + """publish/zips.run() must never build/upload a plugin folder whose name + failed the validate-time safe-name allowlist, even if it somehow reached + `main` (defense in depth alongside the detect.py gate).""" + monkeypatch.chdir(tmp_path) + + safe_dir = tmp_path / "plugins" / "good-plugin" + safe_dir.mkdir(parents=True) + (safe_dir / "plugin.json").write_text(json.dumps({ + "name": "Good Plugin", "version": "1.0.0", + }), encoding="utf-8") + + # Unsafe-named folder with no plugin.json - if run() ever tried to process + # it (instead of skipping via the safe-name check), this would raise + # FileNotFoundError and fail the test. + unsafe_dir = tmp_path / "plugins" / "Bad_Plugin" + unsafe_dir.mkdir(parents=True) + + monkeypatch.setattr(gh, "release_exists", lambda *a, **k: True) + + rc = zips.run("main", "org/repo", str(tmp_path / "build_meta")) + + assert rc == 0 + changed = (tmp_path / "changed_plugins.txt").read_text(encoding="utf-8") + assert "Bad_Plugin" not in changed From 3e61c25623bfb37cec735696c8e7c204bf4fbdde Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sun, 2 Aug 2026 09:38:32 -0400 Subject: [PATCH 06/13] feat(ci): respect CodeQL suppressions and stop truncating findings Port upstream PR #208 into pluginctl: SARIF results with a recognized inline `codeql[...]` suppression no longer count toward blocking/medium/low CodeQL findings, get their own "suppressed findings" table in the PR comment, and gain a CodeQL Suppression Used label that forces manual review before merge. Also drops the 150-char truncation on finding descriptions. --- .github/pluginctl/fixtures/sample.sarif | 6 +++ .../src/pluginctl/integrations/automerge.py | 2 +- .../src/pluginctl/validate/report.py | 20 ++++++-- .../pluginctl/src/pluginctl/validate/sarif.py | 49 +++++++++++++++++-- .github/pluginctl/tests/test_integrations.py | 5 ++ .github/pluginctl/tests/test_report.py | 16 ++++++ .github/pluginctl/tests/test_sarif.py | 22 +++++++-- .github/workflows/_codeql-scan.yml | 38 ++++++++++++++ .github/workflows/validate-plugin.yml | 8 +++ 9 files changed, 154 insertions(+), 12 deletions(-) diff --git a/.github/pluginctl/fixtures/sample.sarif b/.github/pluginctl/fixtures/sample.sarif index 77b431bf..40920cd5 100644 --- a/.github/pluginctl/fixtures/sample.sarif +++ b/.github/pluginctl/fixtures/sample.sarif @@ -36,6 +36,12 @@ "ruleId": "py/unused-import", "message": {"text": "Unused import."}, "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/x.py"}, "region": {"startLine": 1}}}] + }, + { + "ruleId": "py/sql-injection", + "message": {"text": "Suppressed via inline codeql[py/sql-injection] comment."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/other.py"}, "region": {"startLine": 9}}}], + "suppressions": [{"kind": "inSource"}] } ] } diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py index 57931066..e1f42056 100644 --- a/.github/pluginctl/src/pluginctl/integrations/automerge.py +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -17,7 +17,7 @@ from ..core import actions, gh from ..core.git import run as _git -BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE") +BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE", "CodeQL Suppression Used") @dataclass diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 83490ae8..41a53198 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -91,7 +91,8 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, fragment_failed: bool, test_result: str = "skipped", codeql_findings: str = "", codeql_medium_findings: str = "", codeql_low_findings: str = "", clamav_findings: str = "", - other_plugins_section: str = "") -> str: + other_plugins_section: str = "", codeql_suppressed: str = "", + codeql_suppressed_findings: str = "") -> str: L: list[str] = [] overall_failed = fragment_failed if title_valid and title_valid != "true": @@ -200,7 +201,7 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, L.append("") if _needs_separator(codeql_result, codeql_mediums, codeql_lows, - codeql_unscanned_langs, clamav_result): + codeql_unscanned_langs, clamav_result, codeql_suppressed): L.append("") L.append("---") L.append("") @@ -243,6 +244,15 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, L.append("") L.append("") + if codeql_suppressed and codeql_suppressed != "0" and codeql_result != "skipped": + L.append("") + L.append(f"**{codeql_suppressed} finding(s) suppressed via inline `codeql[...]` " + "comment** - requires maintainer review before merging; auto-merge is " + "blocked for this PR.") + L.append("") + if codeql_suppressed_findings: + L.append(codeql_suppressed_findings.rstrip("\n")) + if codeql_result == "skipped" and codeql_unscanned_langs: display = codeql_unscanned_langs.replace(",", " ") L.append("") @@ -300,13 +310,15 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, def _needs_separator(codeql_result, codeql_mediums, codeql_lows, - codeql_unscanned_langs, clamav_result) -> bool: + codeql_unscanned_langs, clamav_result, codeql_suppressed="") -> bool: if codeql_result and codeql_result != "skipped" and codeql_result != "success": return True if codeql_mediums and codeql_mediums != "0" and codeql_result != "skipped": return True if codeql_lows and codeql_lows != "0" and codeql_result != "skipped": return True + if codeql_suppressed and codeql_suppressed != "0" and codeql_result != "skipped": + return True if codeql_result == "skipped" and codeql_unscanned_langs: return True if codeql_result != "skipped" and codeql_result and codeql_unscanned_langs: @@ -405,6 +417,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, codeql_mediums=os.environ.get("CODEQL_MEDIUMS", ""), codeql_lows=os.environ.get("CODEQL_LOWS", ""), codeql_unscanned_langs=os.environ.get("CODEQL_UNSCANNED_LANGS", ""), + codeql_suppressed=os.environ.get("CODEQL_SUPPRESSED", ""), clamav_result=os.environ.get("CLAMAV_RESULT", ""), clamav_infected=os.environ.get("CLAMAV_INFECTED", ""), title_valid=os.environ.get("TITLE_VALID", ""), @@ -418,6 +431,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, codeql_low_findings=_read("codeql-low-findings/codeql-low-findings.md"), clamav_findings=_read("clamav-findings/clamav-findings.md"), other_plugins_section=other_section, + codeql_suppressed_findings=_read("codeql-suppressed-findings/codeql-suppressed-findings.md"), ) with open("pr_comment.txt", "w", encoding="utf-8") as fh: diff --git a/.github/pluginctl/src/pluginctl/validate/sarif.py b/.github/pluginctl/src/pluginctl/validate/sarif.py index 6c9b7ff2..2be3d9e3 100644 --- a/.github/pluginctl/src/pluginctl/validate/sarif.py +++ b/.github/pluginctl/src/pluginctl/validate/sarif.py @@ -20,7 +20,6 @@ from dataclasses import dataclass ZWSP = "​" -ELLIPSIS = "…" # jq: \[(?[^\]]+)\][(][0-9]+[)] -> .c (strip markdown [text](123) links) _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\([0-9]+\)") @@ -82,11 +81,21 @@ def severity_of(result: dict, secmap: dict[str, float]) -> float: return secmap.get(_rule_id(result), 0.0) +def is_suppressed(result: dict) -> bool: + """True when CodeQL's own engine recognized and applied a suppression + + (e.g. a correctly-placed inline ``codeql[...]`` comment) - trust that + signal and never let the result count as blocking/medium/low. + """ + return len(result.get("suppressions") or []) > 0 + + @dataclass class Counts: blocking: int = 0 medium: int = 0 low: int = 0 + suppressed: int = 0 total: int = 0 @property @@ -101,6 +110,9 @@ def classify(sarif_objs: list[dict]) -> Counts: secmap = build_secmap(run) for result in (run.get("results") or []): counts.total += 1 + if is_suppressed(result): + counts.suppressed += 1 + continue sev = severity_of(result, secmap) if is_blocking(sev): counts.blocking += 1 @@ -120,8 +132,6 @@ def process_message(text) -> str: msg = _HASH_DIGIT_RE.sub(f"#{ZWSP}", msg) msg = _MD_LINK_RE.sub(r"\1", msg) msg = msg.replace("[", "[").replace("]", "]") - if len(msg) > 150: - msg = msg[:150] + ELLIPSIS return msg @@ -149,6 +159,8 @@ def findings_table(sarif_objs: list[dict], predicate, repo: str, sha: str, for run in obj.get("runs", []): secmap = build_secmap(run) for result in (run.get("results") or []): + if is_suppressed(result): + continue sev = severity_of(result, secmap) if not predicate(sev): continue @@ -164,6 +176,30 @@ def findings_table(sarif_objs: list[dict], predicate, repo: str, sha: str, return "\n".join(lines) + "\n" +def suppressed_findings_table(sarif_objs: list[dict], repo: str, sha: str, + external_prefixes: list[str]) -> str: + """Same shape as :func:`findings_table`, but for suppressed results. + + Selection is by ``.suppressions`` rather than a severity bucket. + """ + lines = ["| Rule | Location | Description |", "|------|----------|-------------|"] + for obj in sarif_objs: + for run in obj.get("runs", []): + for result in (run.get("results") or []): + if not is_suppressed(result): + continue + rid = _rule_id(result) + uri, line = _location(result) + msg = process_message((result.get("message") or {}).get("text")) + is_external = any(uri.startswith(p) for p in external_prefixes) + if uri != "?" and line != "?" and not is_external: + loc = f"[{uri}:{line}](https://github.com/{repo}/blob/{sha}/{uri}#L{line})" + else: + loc = f"{uri}:{line}" + lines.append(f"| `{rid}` | {loc} | {msg} |") + return "\n".join(lines) + "\n" + + # Bucket predicates matching JQ_BLOCKING / JQ_MEDIUM / JQ_LOW. def is_blocking(sev: float) -> bool: return sev >= 7.0 @@ -216,12 +252,14 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], actions.log( f"Found {counts.blocking} high/critical, {counts.medium} medium, " - f"{counts.low} low, and {counts.warnings} other CodeQL result(s)" + f"{counts.low} low, {counts.suppressed} suppressed, and " + f"{counts.warnings} other CodeQL result(s)" ) actions.set_output("codeql_errors", str(counts.blocking)) actions.set_output("codeql_warnings", str(counts.warnings)) actions.set_output("codeql_mediums", str(counts.medium)) actions.set_output("codeql_lows", str(counts.low)) + actions.set_output("codeql_suppressed", str(counts.suppressed)) if counts.blocking > 0 and results_dir: _write("codeql-findings.md", @@ -232,6 +270,9 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], if counts.low > 0: _write("codeql-low-findings.md", findings_table(objs, is_low, repo, sha, external_prefixes)) + if counts.suppressed > 0: + _write("codeql-suppressed-findings.md", + suppressed_findings_table(objs, repo, sha, external_prefixes)) analyze_failed = languages_found and analyze_outcome not in ("success", "") config_error_langs = "" diff --git a/.github/pluginctl/tests/test_integrations.py b/.github/pluginctl/tests/test_integrations.py index 5df9d0ce..3428037d 100644 --- a/.github/pluginctl/tests/test_integrations.py +++ b/.github/pluginctl/tests/test_integrations.py @@ -18,6 +18,11 @@ def test_automerge_blocks_on_quarantine(): assert not d.ok and "QUARANTINE" in d.reason +def test_automerge_blocks_on_codeql_suppression(): + d = automerge.evaluate_labels(["Plugin Update", "CodeQL Suppression Used"], "MERGEABLE") + assert not d.ok and "CodeQL Suppression Used" in d.reason + + def test_automerge_requires_mergeable(): d = automerge.evaluate_labels(["Plugin Update"], "CONFLICTING") assert not d.ok and "mergeable" in d.reason diff --git a/.github/pluginctl/tests/test_report.py b/.github/pluginctl/tests/test_report.py index d4f1a579..bcbeb99f 100644 --- a/.github/pluginctl/tests/test_report.py +++ b/.github/pluginctl/tests/test_report.py @@ -87,6 +87,22 @@ def test_test_suite_skipped_renders_nothing(): assert "Tooling test suite" not in c +def test_codeql_suppressed_block_is_informational_only(): + c = report.build_comment(**_base_kwargs( + codeql_result="success", codeql_suppressed="2", + codeql_suppressed_findings="| Rule | Location | Description |\n")) + assert "**2 finding(s) suppressed via inline `codeql[...]` comment**" in c + assert "requires maintainer review before merging" in c + # informational only - must not flip the overall pass/fail state + assert "## 🎉 All validation checks passed!" in c + + +def test_codeql_suppressed_skipped_when_codeql_skipped(): + c = report.build_comment(**_base_kwargs( + codeql_result="skipped", codeql_suppressed="2")) + assert "suppressed via inline" not in c + + def test_codeql_skipped_notice_no_separator_when_only_success(): c = report.build_comment(**_base_kwargs(codeql_result="success")) # success + no unscanned -> no findings separator, still passes diff --git a/.github/pluginctl/tests/test_sarif.py b/.github/pluginctl/tests/test_sarif.py index d3eb137a..c5896478 100644 --- a/.github/pluginctl/tests/test_sarif.py +++ b/.github/pluginctl/tests/test_sarif.py @@ -23,8 +23,9 @@ def test_classify_buckets(): assert counts.blocking == 1 assert counts.medium == 1 assert counts.low == 2 - assert counts.total == 4 - assert counts.warnings == 3 + assert counts.suppressed == 1 + assert counts.total == 5 + assert counts.warnings == 4 def test_missing_severity_defaults_to_low(): @@ -46,9 +47,9 @@ def test_process_message_zero_width_spaces(): assert "#​4" in out -def test_process_message_truncation(): +def test_process_message_not_truncated(): out = sarif.process_message("a" * 200) - assert len(out) == 151 and out.endswith("…") + assert out == "a" * 200 def test_findings_table_blocking_links_internal(): @@ -65,6 +66,19 @@ def test_findings_table_external_prefix_plain_location(): assert "plugins/demo/main.py:42" in table +def test_findings_table_excludes_suppressed_results(): + table = sarif.findings_table(_load(), lambda sev: True, "org/repo", "abc123", []) + assert "Suppressed via inline codeql" not in table + + +def test_suppressed_findings_table_includes_only_suppressed(): + table = sarif.suppressed_findings_table(_load(), "org/repo", "abc123", []) + assert "Suppressed via inline codeql" in table + assert "plugins/demo/other.py:9" in table + # only the suppressed py/sql-injection result appears, not the blocking one + assert table.count("`py/sql-injection`") == 1 + + def test_compute_status(): assert sarif.compute_status(1, False, True, False) == "failure" assert sarif.compute_status(0, True, True, False) == "failure" diff --git a/.github/workflows/_codeql-scan.yml b/.github/workflows/_codeql-scan.yml index 1b052180..c9fb0c77 100644 --- a/.github/workflows/_codeql-scan.yml +++ b/.github/workflows/_codeql-scan.yml @@ -30,10 +30,14 @@ on: value: ${{ jobs.analyze.outputs.codeql_lows }} codeql_unscanned_langs: value: ${{ jobs.analyze.outputs.codeql_unscanned_langs }} + codeql_suppressed: + value: ${{ jobs.analyze.outputs.codeql_suppressed }} permissions: contents: read security-events: write + pull-requests: write + issues: write env: PYTHONPATH: ${{ github.workspace }}/_trusted/.github/pluginctl/src @@ -49,6 +53,7 @@ jobs: codeql_mediums: ${{ steps.status.outputs.codeql_mediums }} codeql_lows: ${{ steps.status.outputs.codeql_lows }} codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} + codeql_suppressed: ${{ steps.status.outputs.codeql_suppressed }} steps: - name: Checkout PR merge commit for analysis uses: actions/checkout@v6 @@ -153,6 +158,39 @@ jobs: path: codeql-low-findings.md if-no-files-found: ignore + - name: Upload suppressed findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-suppressed-findings + path: codeql-suppressed-findings.md + if-no-files-found: ignore + + - name: Apply/clear CodeQL suppression label + # Purely informational for auto-merge gating (see automerge.py's + # BLOCKING_LABELS) - never fails the job, so a maintainer can still + # merge by hand after reviewing. + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + SUPPRESSED_COUNT: ${{ steps.status.outputs.codeql_suppressed }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + gh label create "CodeQL Suppression Used" \ + --color "FBCA04" \ + --description "PR relies on an inline CodeQL suppression comment - requires maintainer review" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + if [[ "${SUPPRESSED_COUNT:-0}" -gt 0 ]]; then + gh pr edit "$PR_NUMBER" \ + --add-label "CodeQL Suppression Used" \ + --repo "$GITHUB_REPOSITORY" + else + gh pr edit "$PR_NUMBER" \ + --remove-label "CodeQL Suppression Used" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + fi + - name: Fail job if CodeQL found high/error/critical issues if: always() && steps.status.outputs.codeql_status == 'failure' run: exit 1 diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 68007c15..5d76e6f0 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -324,6 +324,13 @@ jobs: path: codeql-low-findings continue-on-error: true + - name: Download CodeQL suppressed findings detail + uses: actions/download-artifact@v8 + with: + name: codeql-suppressed-findings + path: codeql-suppressed-findings + continue-on-error: true + - name: Download ClamAV findings detail uses: actions/download-artifact@v8 with: @@ -342,6 +349,7 @@ jobs: CODEQL_WARNINGS: ${{ needs.codeql-analyze.outputs.codeql_warnings }} CODEQL_MEDIUMS: ${{ needs.codeql-analyze.outputs.codeql_mediums }} CODEQL_LOWS: ${{ needs.codeql-analyze.outputs.codeql_lows }} + CODEQL_SUPPRESSED: ${{ needs.codeql-analyze.outputs.codeql_suppressed }} CODEQL_UNSCANNED_LANGS: ${{ needs.codeql-analyze.outputs.codeql_unscanned_langs }} CLAMAV_RESULT: ${{ needs.clamav-scan.outputs.clamav_status }} CLAMAV_INFECTED: ${{ needs.clamav-scan.outputs.clamav_infected }} From bc78f61a75dcf770ba4bc63efb972f4288abfa03 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 12 Aug 2026 09:13:48 -0400 Subject: [PATCH 07/13] fix(ci): apply CodeQL alert-suppression fix (PR #225/#226) to pluginctl Rebasing onto upstream/main pulled in PR #225's AlertSuppression fix and PR #226's follow-up, but both only touched the old inline CodeQL block in validate-plugin.yml, which the pluginctl rewrite (4973a50) had already replaced with _codeql-scan.yml. Port the same fix into that architecture: point CodeQL init at codeql-config.yml (sourced from the trusted base-branch checkout, not the untrusted PR merge tree) and dedupe repeated data-flow sentences in pluginctl's process_message, matching PR #225's jq logic. --- .github/codeql/codeql-config.yml | 2 +- .github/pluginctl/src/pluginctl/validate/sarif.py | 11 ++++++++++- .github/pluginctl/tests/test_sarif.py | 10 ++++++++++ .github/workflows/_codeql-scan.yml | 10 ++++++++-- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 4551161d..f1b3f467 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,5 +1,5 @@ # Extra CodeQL query applied alongside the `security-extended` suite used by -# validate-plugin.yml. `packs:` is required here (rather than the simpler +# _codeql-scan.yml. `packs:` is required here (rather than the simpler # `packs:` input on codeql-action/init) because that input only works for # single-language analyses, and our workflow can initialize multiple # languages at once. diff --git a/.github/pluginctl/src/pluginctl/validate/sarif.py b/.github/pluginctl/src/pluginctl/validate/sarif.py index 2be3d9e3..68e8fdf5 100644 --- a/.github/pluginctl/src/pluginctl/validate/sarif.py +++ b/.github/pluginctl/src/pluginctl/validate/sarif.py @@ -25,6 +25,8 @@ _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\([0-9]+\)") _HASH_DIGIT_RE = re.compile(r"#(?=[0-9])") _WWW_RE = re.compile(r"www\.") +# jq: splits("(?<=[.!?]) ") -> split into sentences, dedup, rejoin +_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?]) ") def load_sarif_dir(path: str) -> list[dict]: @@ -132,7 +134,14 @@ def process_message(text) -> str: msg = _HASH_DIGIT_RE.sub(f"#{ZWSP}", msg) msg = _MD_LINK_RE.sub(r"\1", msg) msg = msg.replace("[", "[").replace("]", "]") - return msg + # Data-flow queries can repeat the same sentence once per source/flow reaching + # the same sink - collapse repeats while preserving first-seen order. + sentences = _SENTENCE_SPLIT_RE.split(msg) + seen: list[str] = [] + for sentence in sentences: + if sentence not in seen: + seen.append(sentence) + return " ".join(seen) def _location(result: dict) -> tuple[str, str]: diff --git a/.github/pluginctl/tests/test_sarif.py b/.github/pluginctl/tests/test_sarif.py index c5896478..e17d627c 100644 --- a/.github/pluginctl/tests/test_sarif.py +++ b/.github/pluginctl/tests/test_sarif.py @@ -52,6 +52,16 @@ def test_process_message_not_truncated(): assert out == "a" * 200 +def test_process_message_dedupes_repeated_sentences(): + out = sarif.process_message("This value flows here. This value flows here. Then here.") + assert out == "This value flows here. Then here." + + +def test_process_message_no_repeats_untouched(): + out = sarif.process_message("First sentence. Second sentence.") + assert out == "First sentence. Second sentence." + + def test_findings_table_blocking_links_internal(): table = sarif.findings_table(_load(), sarif.is_blocking, "org/repo", "abc123", []) assert "| Rule | Location | Description |" in table diff --git a/.github/workflows/_codeql-scan.yml b/.github/workflows/_codeql-scan.yml index c9fb0c77..c51a0eb3 100644 --- a/.github/workflows/_codeql-scan.yml +++ b/.github/workflows/_codeql-scan.yml @@ -70,7 +70,9 @@ jobs: repository: ${{ github.repository }} ref: ${{ inputs.base_ref }} path: _trusted - sparse-checkout: .github/pluginctl + sparse-checkout: | + .github/pluginctl + .github/codeql sparse-checkout-cone-mode: false - name: Limit checkout to changed plugin folders only @@ -106,7 +108,11 @@ jobs: with: languages: ${{ steps.detect-langs.outputs.languages }} build-mode: none - queries: security-extended + # Query suite (security-extended: all security severities, no quality queries) and + # per-language alert-suppression packs (so inline `codeql[]` comments work) + # both live in this config file - see .github/codeql/codeql-config.yml for why. Sourced + # from the trusted base-branch checkout, never the untrusted PR merge tree. + config-file: _trusted/.github/codeql/codeql-config.yml - name: Perform CodeQL Analysis if: steps.detect-langs.outputs.found == 'true' From 810fea37d81ce6cee90406f18b56a517155a6cf1 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Wed, 12 Aug 2026 10:30:23 -0400 Subject: [PATCH 08/13] fix(ci): prevent script injection via PR title in validate-title The validate-title job interpolated github.event.pull_request.title directly into a run: shell script instead of passing it through env, letting an attacker-controlled PR title break out of the quoted --title argument and execute arbitrary shell in a job with pull-requests/issues write permissions. --- .github/workflows/validate-plugin.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 5d76e6f0..d5225331 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -172,9 +172,11 @@ jobs: - name: Validate PR title format id: check + env: + PR_TITLE: ${{ github.event.pull_request.title }} run: | python -m pluginctl check-title \ - --title "${{ github.event.pull_request.title }}" \ + --title "$PR_TITLE" \ --author "${{ github.event.pull_request.user.login }}" \ --plugin-count "${{ needs.detect-changes.outputs.plugin_count }}" \ --matrix '${{ needs.detect-changes.outputs.matrix }}' From d98413e6f8e64500a9fa30610e899c76db436867 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Mon, 17 Aug 2026 10:59:06 -0400 Subject: [PATCH 09/13] feat(security): add dormant sandbox bypass detection Add custom Python CodeQL queries and a hardcoded feature flag that keeps sandbox-bypass findings inactive until explicitly enabled. When enabled, findings remain informational, apply a maintainer-review label, and block automatic merging without failing PR validation. --- .github/codeql/codeql-config.yml | 1 + .../codeql/queries/python/BuiltinsMutation.ql | 34 ++++++++++ .github/codeql/queries/python/CtypesUsage.ql | 15 +++++ .../queries/python/FrameGlobalsWrite.ql | 17 +++++ .../python/ObfuscatedDynamicResolution.ql | 16 +++++ .../queries/python/SubclassGadgetBroad.ql | 15 +++++ .../queries/python/SubclassGadgetExact.ql | 27 ++++++++ .../codeql/queries/python/SysModulesTamper.ql | 45 +++++++++++++ .github/codeql/queries/python/qlpack.yml | 4 ++ .../tests/CtypesUsage/CtypesUsage.qlref | 1 + .../queries/python/tests/CtypesUsage/bad.py | 3 + .../queries/python/tests/CtypesUsage/good.py | 3 + .../codeql/queries/python/tests/qlpack.yml | 5 ++ .github/pluginctl/README.md | 3 +- .github/pluginctl/fixtures/sample.sarif | 15 ++++- .../pluginctl/src/pluginctl/feature_flags.py | 5 ++ .../src/pluginctl/integrations/automerge.py | 3 +- .../src/pluginctl/validate/report.py | 29 ++++++-- .../pluginctl/src/pluginctl/validate/sarif.py | 66 +++++++++++++++++-- .github/pluginctl/tests/test_integrations.py | 5 ++ .github/pluginctl/tests/test_report.py | 23 +++++++ .github/pluginctl/tests/test_sarif.py | 33 ++++++++++ .github/workflows/_codeql-scan.yml | 36 ++++++++++ .github/workflows/codeql-query-tests.yml | 26 ++++++++ .github/workflows/validate-plugin.yml | 8 +++ CONTRIBUTING.md | 2 +- docs/plugin-security.md | 38 +++++++++++ 27 files changed, 463 insertions(+), 15 deletions(-) create mode 100644 .github/codeql/queries/python/BuiltinsMutation.ql create mode 100644 .github/codeql/queries/python/CtypesUsage.ql create mode 100644 .github/codeql/queries/python/FrameGlobalsWrite.ql create mode 100644 .github/codeql/queries/python/ObfuscatedDynamicResolution.ql create mode 100644 .github/codeql/queries/python/SubclassGadgetBroad.ql create mode 100644 .github/codeql/queries/python/SubclassGadgetExact.ql create mode 100644 .github/codeql/queries/python/SysModulesTamper.ql create mode 100644 .github/codeql/queries/python/qlpack.yml create mode 100644 .github/codeql/queries/python/tests/CtypesUsage/CtypesUsage.qlref create mode 100644 .github/codeql/queries/python/tests/CtypesUsage/bad.py create mode 100644 .github/codeql/queries/python/tests/CtypesUsage/good.py create mode 100644 .github/codeql/queries/python/tests/qlpack.yml create mode 100644 .github/pluginctl/src/pluginctl/feature_flags.py create mode 100644 .github/workflows/codeql-query-tests.yml create mode 100644 docs/plugin-security.md diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index f1b3f467..377aae7e 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -17,6 +17,7 @@ queries: packs: python: - codeql/python-queries:AlertSuppression.ql + - .github/codeql/queries/python javascript: - codeql/javascript-queries:AlertSuppression.ql go: diff --git a/.github/codeql/queries/python/BuiltinsMutation.ql b/.github/codeql/queries/python/BuiltinsMutation.ql new file mode 100644 index 00000000..d313ae43 --- /dev/null +++ b/.github/codeql/queries/python/BuiltinsMutation.ql @@ -0,0 +1,34 @@ +/** + * @name Plugin mutation of builtins + * @description Mutating builtins can alter import and identity behavior outside a plugin sandbox. + * @kind problem + * @problem.severity warning + * @precision high + * @id plugin/sandbox-bypass/builtins-mutation + * @tags security external/cwe/cwe-693 + */ + +import python + +predicate builtinsObject(Expr expression) { + exists(Name builtins | expression = builtins and builtins.getId() = "__builtins__") +} + +from Expr result +where + exists(AssignStmt assignment, Expr target | + assignment.getTarget() = target and + ( + exists(Attribute attribute | target = attribute and builtinsObject(attribute.getObject())) or + exists(Subscript subscript | target = subscript and builtinsObject(subscript.getObject())) + ) and + result = target + ) + or + exists(Call call, Name setattr | + call.getFunc() = setattr and + setattr.getId() = "setattr" and + builtinsObject(call.getArg(0)) and + result = call + ) +select result, "Mutating __builtins__ can bypass a Python-level plugin sandbox." diff --git a/.github/codeql/queries/python/CtypesUsage.ql b/.github/codeql/queries/python/CtypesUsage.ql new file mode 100644 index 00000000..4d08d524 --- /dev/null +++ b/.github/codeql/queries/python/CtypesUsage.ql @@ -0,0 +1,15 @@ +/** + * @name Plugin use of ctypes + * @description Native ctypes access can bypass Python-level plugin sandboxing. + * @kind problem + * @problem.severity warning + * @precision high + * @id plugin/sandbox-bypass/ctypes-usage + * @tags security external/cwe/cwe-693 + */ + +import python + +from Import imp +where imp.getAnImportedModuleName() = "ctypes" +select imp, "ctypes can load native code outside the plugin sandbox." diff --git a/.github/codeql/queries/python/FrameGlobalsWrite.ql b/.github/codeql/queries/python/FrameGlobalsWrite.ql new file mode 100644 index 00000000..45f178e7 --- /dev/null +++ b/.github/codeql/queries/python/FrameGlobalsWrite.ql @@ -0,0 +1,17 @@ +/** + * @name Plugin writes frame globals or locals + * @description Writing another frame's globals or locals can spoof a frame-based plugin identity check. + * @kind problem + * @problem.severity warning + * @precision medium + * @id plugin/sandbox-bypass/frame-globals-write + * @tags security external/cwe/cwe-693 + */ + +import python + +from AssignStmt assignment, Subscript target, Attribute frameNamespace +where assignment.getTarget() = target and + target.getObject() = frameNamespace and + frameNamespace.getName() in ["f_globals", "f_locals"] +select target, "Writing frame globals or locals can bypass a frame-based plugin sandbox check." diff --git a/.github/codeql/queries/python/ObfuscatedDynamicResolution.ql b/.github/codeql/queries/python/ObfuscatedDynamicResolution.ql new file mode 100644 index 00000000..0ed49a9d --- /dev/null +++ b/.github/codeql/queries/python/ObfuscatedDynamicResolution.ql @@ -0,0 +1,16 @@ +/** + * @name Plugin dynamic code resolution + * @description Dynamic evaluation can obscure module access that bypasses a plugin sandbox. + * @kind problem + * @problem.severity warning + * @precision medium + * @id plugin/sandbox-bypass/obfuscated-dynamic-resolution + * @tags security external/cwe/cwe-693 + */ + +import python + +from Call call, Name function +where call.getFunc() = function and + function.getId() in ["eval", "exec"] +select call, "Dynamic evaluation can obscure an attempted plugin sandbox bypass." diff --git a/.github/codeql/queries/python/SubclassGadgetBroad.ql b/.github/codeql/queries/python/SubclassGadgetBroad.ql new file mode 100644 index 00000000..a9857fcd --- /dev/null +++ b/.github/codeql/queries/python/SubclassGadgetBroad.ql @@ -0,0 +1,15 @@ +/** + * @name Plugin sandbox introspection gadget + * @description Python introspection attributes can expose classes or globals outside a plugin sandbox. + * @kind problem + * @problem.severity warning + * @precision medium + * @id plugin/sandbox-bypass/subclass-gadget-broad + * @tags security external/cwe/cwe-693 + */ + +import python + +from Attribute attribute +where attribute.getName() in ["__subclasses__", "__globals__", "__mro__"] +select attribute, "This introspection attribute can be used to escape a Python-level sandbox." diff --git a/.github/codeql/queries/python/SubclassGadgetExact.ql b/.github/codeql/queries/python/SubclassGadgetExact.ql new file mode 100644 index 00000000..526cf8b1 --- /dev/null +++ b/.github/codeql/queries/python/SubclassGadgetExact.ql @@ -0,0 +1,27 @@ +/** + * @name Plugin subclass enumeration gadget + * @description Enumerating subclasses is a common Python sandbox escape primitive. + * @kind problem + * @problem.severity warning + * @precision high + * @id plugin/sandbox-bypass/subclass-gadget-exact + * @tags security external/cwe/cwe-693 + */ + +import python + +from Call call, Attribute subclasses +where call.getFunc() = subclasses and + subclasses.getName() = "__subclasses__" and + ( + subclasses.getObject().(Name).getId() in ["object", "type"] + or + exists(Subscript index, Attribute bases, Attribute classAttribute | + subclasses.getObject() = index and + index.getObject() = bases and + bases.getName() in ["__bases__", "__mro__"] and + bases.getObject() = classAttribute and + classAttribute.getName() = "__class__" + ) + ) +select call, "Subclass enumeration can expose objects outside the plugin sandbox." diff --git a/.github/codeql/queries/python/SysModulesTamper.ql b/.github/codeql/queries/python/SysModulesTamper.ql new file mode 100644 index 00000000..f45928ed --- /dev/null +++ b/.github/codeql/queries/python/SysModulesTamper.ql @@ -0,0 +1,45 @@ +/** + * @name Plugin tampering with sys.modules + * @description Changing sys.modules can replace or remove sandbox-wrapped modules. + * @kind problem + * @problem.severity warning + * @precision medium + * @id plugin/sandbox-bypass/sys-modules-tamper + * @tags security external/cwe/cwe-693 + */ + +import python + +predicate sensitiveModuleName(Expr expression) { + exists(StringLiteral literal | + expression = literal and + literal.getText() in ["subprocess", "socket", "os", "urllib.request", "requests", + "apps.plugins", "apps.plugins.sandbox"] + ) +} + +predicate sysModules(Attribute attribute) { + exists(Name sys | + attribute.getObject() = sys and sys.getId() = "sys" and attribute.getName() = "modules" + ) +} + +from Expr result +where + exists(AssignStmt assignment, Subscript target, Attribute modules | + assignment.getTarget() = target and + target.getObject() = modules and + sysModules(modules) and + sensitiveModuleName(target.getIndex()) and + result = target + ) + or + exists(Call call, Attribute pop, Attribute modules | + call.getFunc() = pop and + pop.getName() = "pop" and + pop.getObject() = modules and + sysModules(modules) and + sensitiveModuleName(call.getArg(0)) and + result = call + ) +select result, "Changing this sys.modules entry can bypass sandbox-wrapped imports." diff --git a/.github/codeql/queries/python/qlpack.yml b/.github/codeql/queries/python/qlpack.yml new file mode 100644 index 00000000..af29657c --- /dev/null +++ b/.github/codeql/queries/python/qlpack.yml @@ -0,0 +1,4 @@ +name: dispatcharr/plugin-sandbox-bypass-queries +version: 0.1.0 +dependencies: + codeql/python-all: "*" diff --git a/.github/codeql/queries/python/tests/CtypesUsage/CtypesUsage.qlref b/.github/codeql/queries/python/tests/CtypesUsage/CtypesUsage.qlref new file mode 100644 index 00000000..6db78663 --- /dev/null +++ b/.github/codeql/queries/python/tests/CtypesUsage/CtypesUsage.qlref @@ -0,0 +1 @@ +CtypesUsage.ql diff --git a/.github/codeql/queries/python/tests/CtypesUsage/bad.py b/.github/codeql/queries/python/tests/CtypesUsage/bad.py new file mode 100644 index 00000000..7a36a85b --- /dev/null +++ b/.github/codeql/queries/python/tests/CtypesUsage/bad.py @@ -0,0 +1,3 @@ +import ctypes + +ctypes.CDLL("libexample.so") diff --git a/.github/codeql/queries/python/tests/CtypesUsage/good.py b/.github/codeql/queries/python/tests/CtypesUsage/good.py new file mode 100644 index 00000000..3e416262 --- /dev/null +++ b/.github/codeql/queries/python/tests/CtypesUsage/good.py @@ -0,0 +1,3 @@ +import json + +json.dumps({"safe": True}) diff --git a/.github/codeql/queries/python/tests/qlpack.yml b/.github/codeql/queries/python/tests/qlpack.yml new file mode 100644 index 00000000..04ddb080 --- /dev/null +++ b/.github/codeql/queries/python/tests/qlpack.yml @@ -0,0 +1,5 @@ +name: dispatcharr/plugin-sandbox-bypass-query-tests +dependencies: + dispatcharr/plugin-sandbox-bypass-queries: "*" +extractor: python +tests: . diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md index 10fbf6a2..825f0738 100644 --- a/.github/pluginctl/README.md +++ b/.github/pluginctl/README.md @@ -12,7 +12,8 @@ runner; there is no build step. ``` src/pluginctl/ - cli.py argparse dispatch (one subcommand per verb) + cli.py argparse dispatch (one subcommand per verb) + feature_flags.py hardcoded, reviewed rollout switches core/ shared infra: actions, git, gh, jsonio, hashing, timeutil, version validate/ PR-time pipeline: detect, title, labels, gate, sarif, langs, clamav, plugin (per-plugin checks), report diff --git a/.github/pluginctl/fixtures/sample.sarif b/.github/pluginctl/fixtures/sample.sarif index 40920cd5..e18cf463 100644 --- a/.github/pluginctl/fixtures/sample.sarif +++ b/.github/pluginctl/fixtures/sample.sarif @@ -7,7 +7,9 @@ {"id": "py/sql-injection", "properties": {"security-severity": "8.8"}}, {"id": "py/weak-hash", "properties": {"security-severity": "6.5"}}, {"id": "py/clear-text-logging", "properties": {"security-severity": "5.0"}}, - {"id": "py/unused-import", "properties": {}} + {"id": "py/unused-import", "properties": {}}, + {"id": "plugin/sandbox-bypass/ctypes-usage", "properties": {}}, + {"id": "plugin/sandbox-bypass/builtins-mutation", "properties": {}} ] }, "extensions": [ @@ -42,6 +44,17 @@ "message": {"text": "Suppressed via inline codeql[py/sql-injection] comment."}, "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/other.py"}, "region": {"startLine": 9}}}], "suppressions": [{"kind": "inSource"}] + }, + { + "ruleId": "plugin/sandbox-bypass/ctypes-usage", + "message": {"text": "ctypes can bypass the plugin sandbox."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/native.py"}, "region": {"startLine": 12}}}] + }, + { + "ruleId": "plugin/sandbox-bypass/builtins-mutation", + "message": {"text": "Suppressed sandbox finding."}, + "locations": [{"physicalLocation": {"artifactLocation": {"uri": "plugins/demo/builtins.py"}, "region": {"startLine": 4}}}], + "suppressions": [{"kind": "inSource"}] } ] } diff --git a/.github/pluginctl/src/pluginctl/feature_flags.py b/.github/pluginctl/src/pluginctl/feature_flags.py new file mode 100644 index 00000000..b37fc967 --- /dev/null +++ b/.github/pluginctl/src/pluginctl/feature_flags.py @@ -0,0 +1,5 @@ +"""Hardcoded rollout switches for trusted pluginctl automation.""" + +# Flip only after the sandbox-bypass query pack has completed its observation period. +# This is deliberately not environment configurable, so PR code cannot enable it. +SANDBOX_BYPASS_DETECTION = False diff --git a/.github/pluginctl/src/pluginctl/integrations/automerge.py b/.github/pluginctl/src/pluginctl/integrations/automerge.py index e1f42056..1bb746ea 100644 --- a/.github/pluginctl/src/pluginctl/integrations/automerge.py +++ b/.github/pluginctl/src/pluginctl/integrations/automerge.py @@ -17,7 +17,8 @@ from ..core import actions, gh from ..core.git import run as _git -BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE", "CodeQL Suppression Used") +BLOCKING_LABELS = ("New Plugin", "Repo Update", "Invalid", "QUARANTINE", "CodeQL Suppression Used", + "Sandbox Bypass Detected") @dataclass diff --git a/.github/pluginctl/src/pluginctl/validate/report.py b/.github/pluginctl/src/pluginctl/validate/report.py index 41a53198..c324cec4 100644 --- a/.github/pluginctl/src/pluginctl/validate/report.py +++ b/.github/pluginctl/src/pluginctl/validate/report.py @@ -89,10 +89,11 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, clamav_result: str, clamav_infected: str, title_valid: str, title_feedback: str, title_suggestion: str, repository: str, fragment_failed: bool, test_result: str = "skipped", - codeql_findings: str = "", codeql_medium_findings: str = "", - codeql_low_findings: str = "", clamav_findings: str = "", - other_plugins_section: str = "", codeql_suppressed: str = "", - codeql_suppressed_findings: str = "") -> str: + codeql_findings: str = "", codeql_medium_findings: str = "", + codeql_low_findings: str = "", clamav_findings: str = "", + other_plugins_section: str = "", codeql_suppressed: str = "", + codeql_suppressed_findings: str = "", codeql_sandbox_bypass: str = "", + codeql_sandbox_findings: str = "") -> str: L: list[str] = [] overall_failed = fragment_failed if title_valid and title_valid != "true": @@ -201,7 +202,8 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, L.append("") if _needs_separator(codeql_result, codeql_mediums, codeql_lows, - codeql_unscanned_langs, clamav_result, codeql_suppressed): + codeql_unscanned_langs, clamav_result, codeql_suppressed, + codeql_sandbox_bypass): L.append("") L.append("---") L.append("") @@ -215,6 +217,16 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, if clamav_findings: L.append(clamav_findings.rstrip("\n")) + if (codeql_sandbox_bypass and codeql_sandbox_bypass != "0" + and codeql_result != "skipped"): + L.append("") + L.append(f"**CodeQL found {codeql_sandbox_bypass} plugin sandbox-bypass finding(s)**") + L.append("These findings are informational for CI status, but require maintainer review. " + "The `Sandbox Bypass Detected` label blocks automatic merge.") + L.append("") + if codeql_sandbox_findings: + L.append(codeql_sandbox_findings.rstrip("\n")) + if codeql_result and codeql_result not in ("skipped", "success"): L.append("") overall_failed = True @@ -310,7 +322,8 @@ def build_comment(*, plugin_count: str, close_pr: bool, close_reason: str, def _needs_separator(codeql_result, codeql_mediums, codeql_lows, - codeql_unscanned_langs, clamav_result, codeql_suppressed="") -> bool: + codeql_unscanned_langs, clamav_result, codeql_suppressed="", + codeql_sandbox_bypass="") -> bool: if codeql_result and codeql_result != "skipped" and codeql_result != "success": return True if codeql_mediums and codeql_mediums != "0" and codeql_result != "skipped": @@ -319,6 +332,8 @@ def _needs_separator(codeql_result, codeql_mediums, codeql_lows, return True if codeql_suppressed and codeql_suppressed != "0" and codeql_result != "skipped": return True + if codeql_sandbox_bypass and codeql_sandbox_bypass != "0" and codeql_result != "skipped": + return True if codeql_result == "skipped" and codeql_unscanned_langs: return True if codeql_result != "skipped" and codeql_result and codeql_unscanned_langs: @@ -418,6 +433,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, codeql_lows=os.environ.get("CODEQL_LOWS", ""), codeql_unscanned_langs=os.environ.get("CODEQL_UNSCANNED_LANGS", ""), codeql_suppressed=os.environ.get("CODEQL_SUPPRESSED", ""), + codeql_sandbox_bypass=os.environ.get("CODEQL_SANDBOX_BYPASS", ""), clamav_result=os.environ.get("CLAMAV_RESULT", ""), clamav_infected=os.environ.get("CLAMAV_INFECTED", ""), title_valid=os.environ.get("TITLE_VALID", ""), @@ -432,6 +448,7 @@ def run(pr_number: str, pr_author: str, plugin_count: str, close_pr: bool, clamav_findings=_read("clamav-findings/clamav-findings.md"), other_plugins_section=other_section, codeql_suppressed_findings=_read("codeql-suppressed-findings/codeql-suppressed-findings.md"), + codeql_sandbox_findings=_read("codeql-sandbox-findings/codeql-sandbox-findings.md"), ) with open("pr_comment.txt", "w", encoding="utf-8") as fh: diff --git a/.github/pluginctl/src/pluginctl/validate/sarif.py b/.github/pluginctl/src/pluginctl/validate/sarif.py index 68e8fdf5..2eec7c58 100644 --- a/.github/pluginctl/src/pluginctl/validate/sarif.py +++ b/.github/pluginctl/src/pluginctl/validate/sarif.py @@ -19,7 +19,10 @@ import re from dataclasses import dataclass +from .. import feature_flags + ZWSP = "​" +SANDBOX_BYPASS_PREFIX = "plugin/sandbox-bypass/" # jq: \[(?[^\]]+)\][(][0-9]+[)] -> .c (strip markdown [text](123) links) _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\([0-9]+\)") @@ -92,12 +95,19 @@ def is_suppressed(result: dict) -> bool: return len(result.get("suppressions") or []) > 0 +def is_sandbox_bypass_rule(rule_id: str) -> bool: + """True for rules provided by the dormant plugin sandbox query pack.""" + return rule_id.startswith(SANDBOX_BYPASS_PREFIX) + + @dataclass class Counts: blocking: int = 0 medium: int = 0 low: int = 0 suppressed: int = 0 + sandbox_bypass: int = 0 + sandbox_bypass_detected: int = 0 total: int = 0 @property @@ -111,10 +121,17 @@ def classify(sarif_objs: list[dict]) -> Counts: for run in obj.get("runs", []): secmap = build_secmap(run) for result in (run.get("results") or []): + is_sandbox = is_sandbox_bypass_rule(_rule_id(result)) + if is_sandbox and not feature_flags.SANDBOX_BYPASS_DETECTION: + continue counts.total += 1 + if is_sandbox: + counts.sandbox_bypass_detected += 1 if is_suppressed(result): counts.suppressed += 1 continue + if is_sandbox: + counts.sandbox_bypass += 1 sev = severity_of(result, secmap) if is_blocking(sev): counts.blocking += 1 @@ -156,7 +173,7 @@ def _location(result: dict) -> tuple[str, str]: def findings_table(sarif_objs: list[dict], predicate, repo: str, sha: str, - external_prefixes: list[str]) -> str: + external_prefixes: list[str], *, exclude_sandbox_bypass: bool = False) -> str: """Render the ``| Rule | Location | Description |`` markdown table. ``predicate(sev)`` selects the severity bucket. Locations for files sourced @@ -170,10 +187,38 @@ def findings_table(sarif_objs: list[dict], predicate, repo: str, sha: str, for result in (run.get("results") or []): if is_suppressed(result): continue + rid = _rule_id(result) + if is_sandbox_bypass_rule(rid): + if not feature_flags.SANDBOX_BYPASS_DETECTION or exclude_sandbox_bypass: + continue sev = severity_of(result, secmap) if not predicate(sev): continue + uri, line = _location(result) + msg = process_message((result.get("message") or {}).get("text")) + is_external = any(uri.startswith(p) for p in external_prefixes) + if uri != "?" and line != "?" and not is_external: + loc = f"[{uri}:{line}](https://github.com/{repo}/blob/{sha}/{uri}#L{line})" + else: + loc = f"{uri}:{line}" + lines.append(f"| `{rid}` | {loc} | {msg} |") + return "\n".join(lines) + "\n" + + +def sandbox_findings_table(sarif_objs: list[dict], repo: str, sha: str, + external_prefixes: list[str]) -> str: + """Render unsuppressed custom sandbox-bypass findings only.""" + lines = ["| Rule | Location | Description |", "|------|----------|-------------|"] + if not feature_flags.SANDBOX_BYPASS_DETECTION: + return "\n".join(lines) + "\n" + for obj in sarif_objs: + for run in obj.get("runs", []): + for result in (run.get("results") or []): + if is_suppressed(result): + continue rid = _rule_id(result) + if not is_sandbox_bypass_rule(rid): + continue uri, line = _location(result) msg = process_message((result.get("message") or {}).get("text")) is_external = any(uri.startswith(p) for p in external_prefixes) @@ -198,6 +243,8 @@ def suppressed_findings_table(sarif_objs: list[dict], repo: str, sha: str, if not is_suppressed(result): continue rid = _rule_id(result) + if is_sandbox_bypass_rule(rid) and not feature_flags.SANDBOX_BYPASS_DETECTION: + continue uri, line = _location(result) msg = process_message((result.get("message") or {}).get("text")) is_external = any(uri.startswith(p) for p in external_prefixes) @@ -261,7 +308,8 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], actions.log( f"Found {counts.blocking} high/critical, {counts.medium} medium, " - f"{counts.low} low, {counts.suppressed} suppressed, and " + f"{counts.low} low, {counts.suppressed} suppressed, " + f"{counts.sandbox_bypass} sandbox-bypass, and " f"{counts.warnings} other CodeQL result(s)" ) actions.set_output("codeql_errors", str(counts.blocking)) @@ -269,16 +317,24 @@ def run(results_dir: str, repo: str, sha: str, matrix: list[str], actions.set_output("codeql_mediums", str(counts.medium)) actions.set_output("codeql_lows", str(counts.low)) actions.set_output("codeql_suppressed", str(counts.suppressed)) + actions.set_output("codeql_sandbox_bypass", str(counts.sandbox_bypass)) + actions.set_output("codeql_sandbox_bypass_detected", str(counts.sandbox_bypass_detected)) if counts.blocking > 0 and results_dir: _write("codeql-findings.md", - findings_table(objs, is_blocking, repo, sha, external_prefixes)) + findings_table(objs, is_blocking, repo, sha, external_prefixes, + exclude_sandbox_bypass=True)) if counts.medium > 0: _write("codeql-medium-findings.md", - findings_table(objs, is_medium, repo, sha, external_prefixes)) + findings_table(objs, is_medium, repo, sha, external_prefixes, + exclude_sandbox_bypass=True)) if counts.low > 0: _write("codeql-low-findings.md", - findings_table(objs, is_low, repo, sha, external_prefixes)) + findings_table(objs, is_low, repo, sha, external_prefixes, + exclude_sandbox_bypass=True)) + if counts.sandbox_bypass > 0: + _write("codeql-sandbox-findings.md", + sandbox_findings_table(objs, repo, sha, external_prefixes)) if counts.suppressed > 0: _write("codeql-suppressed-findings.md", suppressed_findings_table(objs, repo, sha, external_prefixes)) diff --git a/.github/pluginctl/tests/test_integrations.py b/.github/pluginctl/tests/test_integrations.py index 3428037d..dd59a585 100644 --- a/.github/pluginctl/tests/test_integrations.py +++ b/.github/pluginctl/tests/test_integrations.py @@ -23,6 +23,11 @@ def test_automerge_blocks_on_codeql_suppression(): assert not d.ok and "CodeQL Suppression Used" in d.reason +def test_automerge_blocks_on_sandbox_bypass_detection(): + d = automerge.evaluate_labels(["Plugin Update", "Sandbox Bypass Detected"], "MERGEABLE") + assert not d.ok and "Sandbox Bypass Detected" in d.reason + + def test_automerge_requires_mergeable(): d = automerge.evaluate_labels(["Plugin Update"], "CONFLICTING") assert not d.ok and "mergeable" in d.reason diff --git a/.github/pluginctl/tests/test_report.py b/.github/pluginctl/tests/test_report.py index bcbeb99f..acd59c59 100644 --- a/.github/pluginctl/tests/test_report.py +++ b/.github/pluginctl/tests/test_report.py @@ -103,6 +103,29 @@ def test_codeql_suppressed_skipped_when_codeql_skipped(): assert "suppressed via inline" not in c +def test_sandbox_bypass_is_informational_and_precedes_high_codeql(): + c = report.build_comment(**_base_kwargs( + codeql_result="failure", codeql_errors="1", codeql_sandbox_bypass="2", + codeql_findings="high findings", codeql_sandbox_findings="sandbox findings")) + assert "## ❌ Validation failed" in c # caused by the ordinary high finding + assert c.index("sandbox findings") < c.index("high findings") + + +def test_sandbox_bypass_alone_does_not_fail_comment(): + c = report.build_comment(**_base_kwargs( + codeql_result="success", codeql_sandbox_bypass="1", + codeql_sandbox_findings="sandbox findings")) + assert "Sandbox Bypass Detected" in c + assert "## 🎉 All validation checks passed!" in c + assert "\n---\n" in c + + +def test_sandbox_bypass_is_hidden_when_codeql_is_skipped(): + c = report.build_comment(**_base_kwargs( + codeql_result="skipped", codeql_sandbox_bypass="1")) + assert "Sandbox Bypass Detected" not in c + + def test_codeql_skipped_notice_no_separator_when_only_success(): c = report.build_comment(**_base_kwargs(codeql_result="success")) # success + no unscanned -> no findings separator, still passes diff --git a/.github/pluginctl/tests/test_sarif.py b/.github/pluginctl/tests/test_sarif.py index e17d627c..aabeef3d 100644 --- a/.github/pluginctl/tests/test_sarif.py +++ b/.github/pluginctl/tests/test_sarif.py @@ -28,6 +28,25 @@ def test_classify_buckets(): assert counts.warnings == 4 +def test_sandbox_bypass_is_dormant_by_default(): + counts = sarif.classify(_load()) + assert counts.sandbox_bypass == 0 + assert counts.sandbox_bypass_detected == 0 + assert counts.total == 5 + + +def test_sandbox_bypass_is_informational_when_enabled(monkeypatch): + monkeypatch.setattr(sarif.feature_flags, "SANDBOX_BYPASS_DETECTION", True) + counts = sarif.classify(_load()) + assert counts.blocking == 1 + assert counts.medium == 1 + assert counts.low == 3 + assert counts.suppressed == 2 + assert counts.sandbox_bypass == 1 + assert counts.sandbox_bypass_detected == 2 + assert counts.total == 7 + + def test_missing_severity_defaults_to_low(): counts = sarif.classify(_load()) # py/unused-import has no security-severity -> 0 -> low bucket @@ -89,6 +108,20 @@ def test_suppressed_findings_table_includes_only_suppressed(): assert table.count("`py/sql-injection`") == 1 +def test_sandbox_findings_table_when_enabled(monkeypatch): + monkeypatch.setattr(sarif.feature_flags, "SANDBOX_BYPASS_DETECTION", True) + table = sarif.sandbox_findings_table(_load(), "org/repo", "abc123", []) + assert "`plugin/sandbox-bypass/ctypes-usage`" in table + assert "Suppressed sandbox finding" not in table + + +def test_generic_low_table_excludes_enabled_sandbox_findings(monkeypatch): + monkeypatch.setattr(sarif.feature_flags, "SANDBOX_BYPASS_DETECTION", True) + table = sarif.findings_table(_load(), sarif.is_low, "org/repo", "abc123", [], + exclude_sandbox_bypass=True) + assert "plugin/sandbox-bypass" not in table + + def test_compute_status(): assert sarif.compute_status(1, False, True, False) == "failure" assert sarif.compute_status(0, True, True, False) == "failure" diff --git a/.github/workflows/_codeql-scan.yml b/.github/workflows/_codeql-scan.yml index c51a0eb3..1e4584de 100644 --- a/.github/workflows/_codeql-scan.yml +++ b/.github/workflows/_codeql-scan.yml @@ -32,6 +32,10 @@ on: value: ${{ jobs.analyze.outputs.codeql_unscanned_langs }} codeql_suppressed: value: ${{ jobs.analyze.outputs.codeql_suppressed }} + codeql_sandbox_bypass: + value: ${{ jobs.analyze.outputs.codeql_sandbox_bypass }} + codeql_sandbox_bypass_detected: + value: ${{ jobs.analyze.outputs.codeql_sandbox_bypass_detected }} permissions: contents: read @@ -54,6 +58,8 @@ jobs: codeql_lows: ${{ steps.status.outputs.codeql_lows }} codeql_unscanned_langs: ${{ steps.status.outputs.codeql_unscanned_langs }} codeql_suppressed: ${{ steps.status.outputs.codeql_suppressed }} + codeql_sandbox_bypass: ${{ steps.status.outputs.codeql_sandbox_bypass }} + codeql_sandbox_bypass_detected: ${{ steps.status.outputs.codeql_sandbox_bypass_detected }} steps: - name: Checkout PR merge commit for analysis uses: actions/checkout@v6 @@ -172,6 +178,14 @@ jobs: path: codeql-suppressed-findings.md if-no-files-found: ignore + - name: Upload sandbox-bypass findings detail for PR comment + if: always() + uses: actions/upload-artifact@v7 + with: + name: codeql-sandbox-findings + path: codeql-sandbox-findings.md + if-no-files-found: ignore + - name: Apply/clear CodeQL suppression label # Purely informational for auto-merge gating (see automerge.py's # BLOCKING_LABELS) - never fails the job, so a maintainer can still @@ -197,6 +211,28 @@ jobs: --repo "$GITHUB_REPOSITORY" 2>/dev/null || true fi + - name: Apply/clear sandbox bypass label + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + SANDBOX_BYPASS_COUNT: ${{ steps.status.outputs.codeql_sandbox_bypass_detected }} + PR_NUMBER: ${{ inputs.pr_number }} + run: | + gh label create "Sandbox Bypass Detected" \ + --color "FBCA04" \ + --description "Plugin sandbox-bypass finding requires maintainer review" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + if [[ "${SANDBOX_BYPASS_COUNT:-0}" -gt 0 ]]; then + gh pr edit "$PR_NUMBER" \ + --add-label "Sandbox Bypass Detected" \ + --repo "$GITHUB_REPOSITORY" + else + gh pr edit "$PR_NUMBER" \ + --remove-label "Sandbox Bypass Detected" \ + --repo "$GITHUB_REPOSITORY" 2>/dev/null || true + fi + - name: Fail job if CodeQL found high/error/critical issues if: always() && steps.status.outputs.codeql_status == 'failure' run: exit 1 diff --git a/.github/workflows/codeql-query-tests.yml b/.github/workflows/codeql-query-tests.yml new file mode 100644 index 00000000..428e1322 --- /dev/null +++ b/.github/workflows/codeql-query-tests.yml @@ -0,0 +1,26 @@ +name: CodeQL Query Tests + +on: + pull_request: + paths: + - .github/codeql/queries/** + push: + branches: [main] + paths: + - .github/codeql/queries/** + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v6 + - uses: github/codeql-action/init@v4 + with: + languages: python + # --learn creates the initial expected output in the ephemeral runner while + # still compiling and evaluating the query against its fixture. + - run: codeql test run --learn --search-path=.github/codeql/queries .github/codeql/queries/python/tests diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index d5225331..043cefd9 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -333,6 +333,13 @@ jobs: path: codeql-suppressed-findings continue-on-error: true + - name: Download CodeQL sandbox-bypass findings detail + uses: actions/download-artifact@v8 + with: + name: codeql-sandbox-findings + path: codeql-sandbox-findings + continue-on-error: true + - name: Download ClamAV findings detail uses: actions/download-artifact@v8 with: @@ -352,6 +359,7 @@ jobs: CODEQL_MEDIUMS: ${{ needs.codeql-analyze.outputs.codeql_mediums }} CODEQL_LOWS: ${{ needs.codeql-analyze.outputs.codeql_lows }} CODEQL_SUPPRESSED: ${{ needs.codeql-analyze.outputs.codeql_suppressed }} + CODEQL_SANDBOX_BYPASS: ${{ needs.codeql-analyze.outputs.codeql_sandbox_bypass }} CODEQL_UNSCANNED_LANGS: ${{ needs.codeql-analyze.outputs.codeql_unscanned_langs }} CLAMAV_RESULT: ${{ needs.clamav-scan.outputs.clamav_status }} CLAMAV_INFECTED: ${{ needs.clamav-scan.outputs.clamav_infected }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c4d2fe3..d1de0be4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -162,7 +162,7 @@ Automated validation runs on every PR and posts a comment with results. The foll | `min_dispatcharr_version` | Must be semver if provided | | `max_dispatcharr_version` | Must be semver and ≥ `min_dispatcharr_version` if both provided | | `repo_url` / `discord_thread` | Must start with `http://` or `https://` if provided | -| CodeQL | Python code is scanned for security issues (blocking). For external plugins, the release ZIP is downloaded and its contents scanned | +| CodeQL | Python code is scanned for security issues (blocking). The repository also includes dormant informational sandbox-bypass checks that require review when enabled. For external plugins, the release ZIP is downloaded and its contents scanned. See [plugin security](docs/plugin-security.md) | | ClamAV | All submitted files are scanned for malware (blocking). For external plugins, the release ZIP is downloaded and scanned | | `source_url` | For external plugins: must be an HTTPS URL with a `{version}` placeholder; artifact must be reachable | | `repo_url` | Required for external plugins | diff --git a/docs/plugin-security.md b/docs/plugin-security.md new file mode 100644 index 00000000..c04979a3 --- /dev/null +++ b/docs/plugin-security.md @@ -0,0 +1,38 @@ +# Plugin Security Checks + +Plugin source is checked before merge in two layers. This repository runs static +analysis while the Dispatcharr application enforces runtime plugin sandboxing. +Neither layer is a proof that a plugin is safe, but together they give reviewers +useful signals before distribution. + +## Sandbox-bypass queries + +The Python CodeQL pack detects shapes associated with attempts to bypass a +Python-level plugin sandbox: + +| Rule | Detection | +|---|---| +| `sys-modules-tamper` | Replacement of sensitive `sys.modules` entries | +| `ctypes-usage` | Native-code access through `ctypes` | +| `builtins-mutation` | Mutation of `__builtins__` | +| `subclass-gadget-exact` | Subclass enumeration escape primitives | +| `subclass-gadget-broad` | Broad introspection access to subclasses, globals, or MRO | +| `frame-globals-write` | Writes through frame globals or locals | +| `obfuscated-dynamic-resolution` | Dynamic code evaluation | + +The feature is shipped disabled while its query behavior is reviewed. Its single +hardcoded switch is `pluginctl.feature_flags.SANDBOX_BYPASS_DETECTION`. When a +maintainer enables it, all of these rules remain informational. They do not +create a high or critical CodeQL result and do not fail the required validation +check. + +An enabled finding applies the `Sandbox Bypass Detected` label and blocks only +automatic merging. A maintainer must review the finding before manually merging +the PR. Removing the detected code removes the label on the next validation run. + +## Suppressions + +Use an inline `codeql[rule-id]` suppression only for a justified exception. A +suppressed sandbox result still applies `Sandbox Bypass Detected`, and the +existing `CodeQL Suppression Used` label also blocks automatic merge. Both labels +require maintainer review. From d3bd7bcb90bacec236ac60b2b0fd256b139a325f Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 22 Aug 2026 18:41:40 -0400 Subject: [PATCH 10/13] fix(ci): invoke CodeQL CLI from init output Use the CodeQL Action's explicit binary path in the custom query test workflow and grant its required read-only security-events permission. --- .github/workflows/codeql-query-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-query-tests.yml b/.github/workflows/codeql-query-tests.yml index 428e1322..6acb0992 100644 --- a/.github/workflows/codeql-query-tests.yml +++ b/.github/workflows/codeql-query-tests.yml @@ -11,6 +11,7 @@ on: permissions: contents: read + security-events: read jobs: test: @@ -18,9 +19,10 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v6 - - uses: github/codeql-action/init@v4 + - id: init + uses: github/codeql-action/init@v4 with: languages: python # --learn creates the initial expected output in the ephemeral runner while # still compiling and evaluating the query against its fixture. - - run: codeql test run --learn --search-path=.github/codeql/queries .github/codeql/queries/python/tests + - run: ${{ steps.init.outputs.codeql-path }} test run --learn --search-path=.github/codeql/queries .github/codeql/queries/python/tests From d53c39c66af19202bbe78c8f4a3d32b969222a21 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 22 Aug 2026 19:04:55 -0400 Subject: [PATCH 11/13] feat(publish): gate split manifest base URLs Restore split metadata and download base URL generation behind a hardcoded disabled rollout flag. Format generated manifest wrappers without changing compact comparison or signing payloads. --- .github/pluginctl/README.md | 3 + .github/pluginctl/src/pluginctl/core/gh.py | 5 + .../pluginctl/src/pluginctl/core/jsonio.py | 20 ++-- .../pluginctl/src/pluginctl/feature_flags.py | 4 + .../src/pluginctl/publish/manifest.py | 51 ++++++++--- .../src/pluginctl/publish/readmes.py | 20 ++-- .github/pluginctl/tests/test_manifest.py | 91 +++++++++++++++++++ .github/pluginctl/tests/test_readmes.py | 11 ++- 8 files changed, 176 insertions(+), 29 deletions(-) diff --git a/.github/pluginctl/README.md b/.github/pluginctl/README.md index 825f0738..b714173e 100644 --- a/.github/pluginctl/README.md +++ b/.github/pluginctl/README.md @@ -78,3 +78,6 @@ PYTHONPATH=.github/pluginctl/src python -m pluginctl \ - `publish/readmes.py` intentionally corrects one old bug: deprecated plugins' version count now comes from the manifest's `versions[]` instead of the removed `zips/` path (which always rendered 0). +- Publishing defaults to the legacy `root_url` manifest format while the shell + publisher remains in service. `SPLIT_MANIFEST_BASE_URLS` is a reviewed, + hardcoded rollout flag for `download_base_url` and `metadata_base_url`. diff --git a/.github/pluginctl/src/pluginctl/core/gh.py b/.github/pluginctl/src/pluginctl/core/gh.py index a3d35c26..ef7f44f0 100644 --- a/.github/pluginctl/src/pluginctl/core/gh.py +++ b/.github/pluginctl/src/pluginctl/core/gh.py @@ -132,6 +132,11 @@ def release_create(tag: str, repo: str, title: str, notes: str, asset: str) -> i "--notes", notes, asset]).returncode +def pages_url(repo: str) -> str: + """GitHub Pages site URL for the repo, or an empty string when unavailable.""" + return api(f"repos/{repo}/pages", jq=".html_url // empty") or "" + + def release_delete(tag: str, repo: str, cleanup_tag: bool = True) -> int: args = ["release", "delete", tag, "--repo", repo, "--yes"] if cleanup_tag: diff --git a/.github/pluginctl/src/pluginctl/core/jsonio.py b/.github/pluginctl/src/pluginctl/core/jsonio.py index d9a1ff28..037797d6 100644 --- a/.github/pluginctl/src/pluginctl/core/jsonio.py +++ b/.github/pluginctl/src/pluginctl/core/jsonio.py @@ -1,11 +1,10 @@ -"""Canonical JSON helpers that reproduce the exact byte output of the V1 jq programs. - -The V1 publish pipeline builds manifests with `jq` and emits them with `jq -c`. -To keep the published `releases`-branch output byte-identical, every manifest -object here is built as an insertion-ordered ``dict`` in the same key order as the -corresponding jq program, nulls are dropped the same way jq's -``with_entries(select(.value != null))`` does, and the compact form is produced -with the same separators jq uses. +"""Canonical JSON helpers for manifest payloads and on-disk wrappers. + +Manifest objects are built as insertion-ordered ``dict`` objects in the same key +order as the corresponding jq program. Nulls are dropped the same way jq's +``with_entries(select(.value != null))`` does, and compact JSON is retained for +stable payload comparisons and GPG signing. Published wrapper files use formatted +JSON so humans can inspect them easily. """ from __future__ import annotations @@ -24,6 +23,11 @@ def dumps(obj: Any) -> str: return json.dumps(obj, separators=(",", ":"), ensure_ascii=False) +def dumps_formatted(obj: Any) -> str: + """Indented JSON for generated files, with a conventional trailing newline.""" + return json.dumps(obj, indent=2, ensure_ascii=False) + "\n" + + def drop_none(obj: dict) -> dict: """Reproduce ``with_entries(select(.value != null))`` for a single object. diff --git a/.github/pluginctl/src/pluginctl/feature_flags.py b/.github/pluginctl/src/pluginctl/feature_flags.py index b37fc967..1094d8c7 100644 --- a/.github/pluginctl/src/pluginctl/feature_flags.py +++ b/.github/pluginctl/src/pluginctl/feature_flags.py @@ -3,3 +3,7 @@ # Flip only after the sandbox-bypass query pack has completed its observation period. # This is deliberately not environment configurable, so PR code cannot enable it. SANDBOX_BYPASS_DETECTION = False + +# Keep legacy root_url manifests until the Python publisher fully replaces the +# shell workflow. This is deliberately not environment configurable. +SPLIT_MANIFEST_BASE_URLS = False diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py index 8940a774..948d2b2a 100644 --- a/.github/pluginctl/src/pluginctl/publish/manifest.py +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -17,6 +17,7 @@ import tempfile from typing import Optional +from .. import feature_flags from ..core import jsonio from ..core.jsonio import drop_none from ..core.timeutil import now_iso @@ -143,14 +144,19 @@ def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, }) -def build_root_manifest(registry_url: str, registry_name: str, root_url: str, - root_entries: list[dict]) -> dict: - return { +def build_root_manifest(registry_url: str, registry_name: str, download_base_url: str, + root_entries: list[dict], metadata_base_url: Optional[str] = None) -> dict: + root = { "registry_url": registry_url, "registry_name": registry_name, - "root_url": root_url, - "plugins": root_entries, } + if metadata_base_url is None: + root["root_url"] = download_base_url + else: + root["download_base_url"] = download_base_url + root["metadata_base_url"] = metadata_base_url + root["plugins"] = root_entries + return root def _num(value) -> float: @@ -178,7 +184,7 @@ def write_manifest_if_changed(dest: str, payload: dict, generated_at: str) -> bo pass wrapper = {"generated_at": generated_at, "manifest": json.loads(new_compact)} with open(dest, "w", encoding="utf-8") as fh: - fh.write(jsonio.dumps(wrapper)) + fh.write(jsonio.dumps_formatted(wrapper)) return True @@ -223,7 +229,7 @@ def sign_manifest(file: str, gpg_key_id: str, passphrase: Optional[str]) -> bool return False data["signature"] = sig with open(file, "w", encoding="utf-8") as fh: - fh.write(jsonio.dumps(data)) + fh.write(jsonio.dumps_formatted(data)) return True @@ -259,6 +265,20 @@ def _canonical(version: str) -> str: return re.sub(r"-[0-9]+$", "", version) +ICON_EXTS = ("png", "svg", "jpg", "webp") + + +def _sync_icon(plugin_dir: str, plugin_name: str) -> None: + """Copy the first supported source logo beside the plugin metadata.""" + import shutil + + for ext in ICON_EXTS: + source = os.path.join(plugin_dir, f"logo.{ext}") + if os.path.isfile(source): + shutil.copy(source, f"metadata/{plugin_name}/logo.{ext}") + return + + def strip_signatures() -> None: """Remove ``.signature`` from every manifest (no key configured or signing failed).""" import glob as _glob @@ -273,7 +293,7 @@ def strip_signatures() -> None: if "signature" in data: del data["signature"] with open(f, "w", encoding="utf-8") as fh: - fh.write(jsonio.dumps(data)) + fh.write(jsonio.dumps_formatted(data)) def generate(source_branch: str, releases_branch: str, repository: str, @@ -285,8 +305,11 @@ def generate(source_branch: str, releases_branch: str, repository: str, generated_at = now_iso() registry_url = f"https://github.com/{repository}" registry_name = repository - root_url = f"https://github.com/{repository}/releases/download" + download_base_url = f"https://github.com/{repository}/releases/download" raw_releases_url = f"https://raw.githubusercontent.com/{repository}/{releases_branch}" + metadata_base_url = "" + if feature_flags.SPLIT_MANIFEST_BASE_URLS: + metadata_base_url = gh.pages_url(repository).rstrip("/") or raw_releases_url key_id, signing_failed = import_gpg_key(os.environ.get("GPG_PRIVATE_KEY", "")) passphrase = os.environ.get("GPG_PASSPHRASE") or None @@ -323,6 +346,8 @@ def sign_if_needed(path: str, written: bool) -> None: existing_manifest_file = f"metadata/{plugin_name}/manifest.json" os.makedirs(f"metadata/{plugin_name}", exist_ok=True) + if feature_flags.SPLIT_MANIFEST_BASE_URLS: + _sync_icon(plugin_dir, plugin_name) existing_manifest = None if os.path.isfile(existing_manifest_file): try: @@ -383,13 +408,17 @@ def sign_if_needed(path: str, written: bool) -> None: if unlisted: continue - manifest_url = f"{raw_releases_url}/metadata/{plugin_name}/manifest.json" + manifest_url = (f"metadata/{plugin_name}/manifest.json" + if feature_flags.SPLIT_MANIFEST_BASE_URLS + else f"{raw_releases_url}/metadata/{plugin_name}/manifest.json") root_entries.append(build_root_entry(raw, plugin_name, latest_metadata, latest_size_kb, min_da, max_da, latest_url, manifest_url)) plugin_count += 1 - inner_root = build_root_manifest(registry_url, registry_name, root_url, root_entries) + inner_root = build_root_manifest( + registry_url, registry_name, download_base_url, root_entries, + metadata_base_url if feature_flags.SPLIT_MANIFEST_BASE_URLS else None) written = write_manifest_if_changed("manifest.json", inner_root, generated_at) sign_if_needed("manifest.json", written) diff --git a/.github/pluginctl/src/pluginctl/publish/readmes.py b/.github/pluginctl/src/pluginctl/publish/readmes.py index eb7cae64..ee0c3243 100644 --- a/.github/pluginctl/src/pluginctl/publish/readmes.py +++ b/.github/pluginctl/src/pluginctl/publish/readmes.py @@ -61,7 +61,7 @@ def _g(data: Optional[dict], key: str, default: str = "") -> str: # Per-plugin README (metadata//README.md) # --------------------------------------------------------------------------- # def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, - root_url: str, repository: str, source_branch: str, + download_base_url: str, repository: str, source_branch: str, last_updated: str, has_readme: bool, source_readme: str = "") -> str: name = _g(plugin_raw, "name") @@ -78,7 +78,7 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, manifest_body = manifest.get("manifest", {}) if manifest else {} latest = manifest_body.get("latest") or {} latest_url_path = latest.get("latest_url") or "" - latest_full_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + latest_full_url = f"{download_base_url}/{latest_url_path}" if download_base_url and latest_url_path else "" L: list[str] = [] L.append("[Back to All Plugins](../../README.md)") @@ -134,7 +134,7 @@ def render_plugin_readme(plugin_name: str, plugin_raw: dict, manifest: dict, if not v: continue url_path = ver.get("url") or "" - full_url = f"{root_url}/{url_path}" if root_url and url_path else "" + full_url = f"{download_base_url}/{url_path}" if download_base_url and url_path else "" commit_sha = ver.get("commit_sha") or "" commit_short = ver.get("commit_sha_short") or "" build_ts = ver.get("build_timestamp") or "" @@ -229,7 +229,7 @@ def table_row(plugin_raw: dict, deprecated_pass: bool) -> str: def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: dict, manifest: Optional[dict], last_updated: str, commit_sha: str, commit_sha_short: str, version_count, repository: str, - source_branch: str, releases_branch: str, root_url: str, + source_branch: str, releases_branch: str, download_base_url: str, has_source_readme: bool) -> str: name = _g(plugin_raw, "name") version = _g(plugin_raw, "version") @@ -245,7 +245,7 @@ def render_plugin_block(*, is_deprecated: bool, plugin_name: str, plugin_raw: di latest_url_path = "" if manifest: latest_url_path = ((manifest.get("manifest") or {}).get("latest") or {}).get("latest_url") or "" - zip_url = f"{root_url}/{latest_url_path}" if root_url and latest_url_path else "" + zip_url = f"{download_base_url}/{latest_url_path}" if download_base_url and latest_url_path else "" source_url = f"https://github.com/{repository}/tree/{source_branch}/plugins/{plugin_name}" readme_url = f"https://github.com/{repository}/blob/{source_branch}/plugins/{plugin_name}/README.md" @@ -312,7 +312,8 @@ def _plugin_last_updated(source_branch: str, plugin_dir: str) -> str: def generate_plugin_readmes(source_branch: str, repository: str) -> None: """Write metadata//README.md for every plugin (plugin-readmes.sh).""" root = _load_json("manifest.json") or {} - root_url = (root.get("manifest") or {}).get("root_url") or "" + root_manifest = root.get("manifest") or {} + download_base_url = root_manifest.get("download_base_url") or root_manifest.get("root_url") or "" for plugin_dir in sorted(glob.glob("plugins/*/")): plugin_name = os.path.basename(plugin_dir.rstrip("/")) plugin_file = os.path.join(plugin_dir, "plugin.json") @@ -328,7 +329,7 @@ def generate_plugin_readmes(source_branch: str, repository: str) -> None: has_readme = os.path.isfile(source_readme_path) source_readme = _read(source_readme_path) if has_readme else "" content = render_plugin_readme( - plugin_name, plugin_raw, manifest, root_url, repository, source_branch, + plugin_name, plugin_raw, manifest, download_base_url, repository, source_branch, last_updated, has_readme, source_readme) with open(f"metadata/{plugin_name}/README.md", "w", encoding="utf-8") as fh: fh.write(content) @@ -337,7 +338,8 @@ def generate_plugin_readmes(source_branch: str, repository: str) -> None: def generate_releases_readme(source_branch: str, releases_branch: str, repository: str) -> None: """Write the root README.md (releases-readme.sh).""" root = _load_json("manifest.json") or {} - root_url = (root.get("manifest") or {}).get("root_url") or "" + root_manifest = root.get("manifest") or {} + download_base_url = root_manifest.get("download_base_url") or root_manifest.get("root_url") or "" raws: dict[str, dict] = {} has_deprecated = False @@ -391,7 +393,7 @@ def emit_block(pname: str, raw: dict, is_deprecated: bool) -> None: manifest=manifest, last_updated=last_updated, commit_sha=commit_sha, commit_sha_short=commit_short, version_count=version_count, repository=repository, source_branch=source_branch, - releases_branch=releases_branch, root_url=root_url, + releases_branch=releases_branch, download_base_url=download_base_url, has_source_readme=has_source_readme)) # Active detailed sections diff --git a/.github/pluginctl/tests/test_manifest.py b/.github/pluginctl/tests/test_manifest.py index f26474f9..d3dd236c 100644 --- a/.github/pluginctl/tests/test_manifest.py +++ b/.github/pluginctl/tests/test_manifest.py @@ -1,6 +1,8 @@ import json from pluginctl.core import jsonio +from pluginctl.core import gh +from pluginctl import feature_flags from pluginctl.publish import manifest as m @@ -101,3 +103,92 @@ def test_root_manifest_compact_matches_jq(tmp_path): capture_output=True, text=True, check=True).stdout.strip() assert compact == jq_out # stable round-trip through real jq assert json.loads(compact)["plugins"][0]["slug"] == "demo" + + +def test_root_manifest_split_base_urls_key_order(): + entry = m.build_root_entry(PLUGIN_RAW, "demo", META, 10, "", "", + "u", "metadata/demo/manifest.json") + root = m.build_root_manifest( + "https://github.com/org/repo", "org/repo", + "https://github.com/org/repo/releases/download", [entry], + "https://org.github.io/repo") + assert list(root) == [ + "registry_url", "registry_name", "download_base_url", + "metadata_base_url", "plugins", + ] + assert "root_url" not in root + assert root["plugins"][0]["manifest_url"] == "metadata/demo/manifest.json" + + +def test_write_manifest_uses_formatted_json_and_ignores_existing_whitespace(tmp_path): + dest = tmp_path / "manifest.json" + payload = {"registry_name": "demo", "plugins": []} + + assert m.write_manifest_if_changed(str(dest), payload, "2026-01-01T00:00:00Z") + rendered = dest.read_text(encoding="utf-8") + assert rendered.startswith("{\n \"generated_at\":") + assert rendered.endswith("\n") + assert json.loads(rendered)["manifest"] == payload + + dest.write_text(json.dumps({"manifest": payload}), encoding="utf-8") + assert not m.write_manifest_if_changed(str(dest), payload, "2026-01-02T00:00:00Z") + assert dest.read_text(encoding="utf-8") == json.dumps({"manifest": payload}) + + +def test_pages_url_returns_empty_string_when_pages_is_unavailable(monkeypatch): + monkeypatch.setattr(gh, "api", lambda *args, **kwargs: None) + assert gh.pages_url("org/repo") == "" + + +def test_sync_icon_prefers_png(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + plugin_dir = tmp_path / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / "logo.png").write_bytes(b"png") + (plugin_dir / "logo.svg").write_bytes(b"svg") + (tmp_path / "metadata" / "demo").mkdir(parents=True) + + m._sync_icon(str(plugin_dir), "demo") + + assert (tmp_path / "metadata" / "demo" / "logo.png").read_bytes() == b"png" + assert not (tmp_path / "metadata" / "demo" / "logo.svg").exists() + + +def test_generate_uses_legacy_manifest_by_default(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + plugin_dir = tmp_path / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / "plugin.json").write_text( + json.dumps({"name": "Demo", "version": "1.0.0", "description": "d"}), + encoding="utf-8") + monkeypatch.setattr(feature_flags, "SPLIT_MANIFEST_BASE_URLS", False) + monkeypatch.setattr(gh, "release_list_tags", lambda repo: []) + monkeypatch.setattr(gh, "pages_url", lambda repo: (_ for _ in ()).throw(AssertionError())) + + assert m.generate("main", "releases", "org/repo", "") == 0 + + root = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))["manifest"] + assert root["root_url"] == "https://github.com/org/repo/releases/download" + assert root["plugins"][0]["manifest_url"].startswith("https://raw.githubusercontent.com/") + + +def test_generate_uses_split_manifest_and_copies_icon_when_enabled(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + plugin_dir = tmp_path / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / "plugin.json").write_text( + json.dumps({"name": "Demo", "version": "1.0.0", "description": "d"}), + encoding="utf-8") + (plugin_dir / "logo.png").write_bytes(b"png") + monkeypatch.setattr(feature_flags, "SPLIT_MANIFEST_BASE_URLS", True) + monkeypatch.setattr(gh, "release_list_tags", lambda repo: []) + monkeypatch.setattr(gh, "pages_url", lambda repo: "https://org.github.io/repo/") + + assert m.generate("main", "releases", "org/repo", "") == 0 + + root = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))["manifest"] + assert root["download_base_url"] == "https://github.com/org/repo/releases/download" + assert root["metadata_base_url"] == "https://org.github.io/repo" + assert "root_url" not in root + assert root["plugins"][0]["manifest_url"] == "metadata/demo/manifest.json" + assert (tmp_path / "metadata" / "demo" / "logo.png").read_bytes() == b"png" diff --git a/.github/pluginctl/tests/test_readmes.py b/.github/pluginctl/tests/test_readmes.py index 6003d9cb..63c537f4 100644 --- a/.github/pluginctl/tests/test_readmes.py +++ b/.github/pluginctl/tests/test_readmes.py @@ -50,6 +50,15 @@ def test_render_plugin_block_uses_version_count(): is_deprecated=True, plugin_name="old", plugin_raw=raw, manifest=None, last_updated="2024-01-01T00:00:00Z", commit_sha="abc", commit_sha_short="abc", version_count=3, repository="org/repo", source_branch="main", - releases_branch="releases", root_url="https://dl", has_source_readme=False) + releases_branch="releases", download_base_url="https://dl", has_source_readme=False) assert "### [Old](https://github.com/org/repo/blob/releases/metadata/old/README.md) (deprecated)" in block assert "- [All Versions (3 available)](./metadata/old)" in block + + +def test_render_plugin_readme_uses_download_base_url(): + content = r.render_plugin_readme( + "demo", {"name": "Demo", "version": "1.0.0", "description": "d"}, + {"manifest": {"latest": {"latest_url": "demo-1.0.0/demo.zip"}, + "versions": [{"version": "1.0.0", "url": "demo-1.0.0/demo.zip"}]}}, + "https://downloads.example", "org/repo", "main", "", False) + assert content.count("https://downloads.example/demo-1.0.0/demo.zip") == 2 From 0b12e20cae5c91083894de9486a4a41f333ec147 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 22 Aug 2026 19:15:27 -0400 Subject: [PATCH 12/13] feat(ci): optimize metadata-only plugin validation Classify fully metadata-only plugin updates during change detection.\nSkip CodeQL for those PRs while retaining ClamAV scans without\nre-downloading external release ZIPs. --- .../src/pluginctl/validate/detect.py | 54 ++++++++++++++-- .../src/pluginctl/validate/plugin.py | 15 +---- .github/pluginctl/tests/test_detect.py | 61 +++++++++++++++++++ .github/workflows/_clamav-scan.yml | 8 ++- .github/workflows/validate-plugin.yml | 4 +- 5 files changed, 121 insertions(+), 21 deletions(-) diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index 9572b7f3..da814343 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -17,6 +17,10 @@ SAFE_NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") PUB_KEY_PATH = ".github/scripts/keys/dispatcharr-plugins.pub" +METADATA_ONLY_FIELDS = { + "description", "repo_url", "discord_thread", "min_dispatcharr_version", + "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", +} def is_safe_name(name: str) -> bool: @@ -44,6 +48,29 @@ def author_has_plugin_permission(pr_author: str, base_json_text: Optional[str]) return author_in_plugin_json(pr_author, data) +def changed_json_fields(old: dict, new: dict) -> set[str]: + """Return manifest keys whose value changed, including additions and removals.""" + return {key for key in old.keys() | new.keys() if old.get(key) != new.get(key)} + + +def is_metadata_only_update(plugin_name: str, base_ref: str, changed_files: list[str]) -> bool: + """Whether an existing plugin only changes approved metadata in plugin.json.""" + plugin_json_path = f"plugins/{plugin_name}/plugin.json" + if changed_files != [plugin_json_path]: + return False + + base_json_text = git.show(f"origin/{base_ref}:{plugin_json_path}") + if base_json_text is None: + return False + try: + with open(plugin_json_path, encoding="utf-8") as fh: + current = json.load(fh) + base = json.loads(base_json_text) + except (OSError, json.JSONDecodeError): + return False + return bool(changed_json_fields(base, current)) and changed_json_fields(base, current) <= METADATA_ONLY_FIELDS + + @dataclass class BlacklistResult: matched: bool = False @@ -94,7 +121,8 @@ def write_access() -> bool: actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", close_reason="", skip_validation="true", outside_violation="false", pub_key_changed="false", - has_new_plugin="false", has_updated_plugin="false") + has_new_plugin="false", has_updated_plugin="false", + metadata_only="false") return 0 merge_base = git.merge_base(f"origin/{base_ref}", "HEAD") @@ -126,14 +154,16 @@ def write_access() -> bool: actions.error(f"Unsafe plugin folder name(s) detected: {', '.join(unsafe_list)}") actions.set_outputs(close_pr="true", close_reason="unsafe-plugin-name", plugin_count="0", matrix="[]", - has_new_plugin="false", has_updated_plugin="false") + has_new_plugin="false", has_updated_plugin="false", + metadata_only="false") return 0 if not plugin_list: actions.error("No valid plugin changes detected in this PR.") actions.set_outputs(close_pr="true", close_reason="no-valid-plugins", plugin_count="0", matrix="[]", - has_new_plugin="false", has_updated_plugin="false") + has_new_plugin="false", has_updated_plugin="false", + metadata_only="false") return 0 plugin_count = len(plugin_list) @@ -146,6 +176,15 @@ def write_access() -> bool: else: has_new_plugin = True + metadata_only = all( + is_metadata_only_update( + plugin, + base_ref, + sorted(f for f in changed if f.startswith(f"plugins/{plugin}/")), + ) + for plugin in plugin_list + ) + is_maintainer = write_access() has_any_permission = is_maintainer if not is_maintainer: @@ -170,7 +209,8 @@ def write_access() -> bool: close_pr="true" if close_pr else "false", skip_validation="false", outside_violation="true" if has_outside_violation else "false", - close_reason=close_reason) + close_reason=close_reason, + metadata_only="true" if metadata_only else "false") if outside_changes: actions.set_output("outside_files", "\n".join(outside_changes)) @@ -189,7 +229,8 @@ def _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) actions.set_outputs(matrix="[]", plugin_count="0", close_pr="false", close_reason="", skip_validation="false", outside_violation="true", - has_new_plugin="false", has_updated_plugin="false") + has_new_plugin="false", has_updated_plugin="false", + metadata_only="false") actions.set_output("outside_files", "\n".join(outside_changes)) return 0 if write_access(): @@ -198,7 +239,8 @@ def _no_plugins(pr_author, outside_changes, has_outside_violation, write_access) close_reason="", skip_validation="true", outside_violation="false", pub_key_changed="true" if pub_key_changed else "false", - has_new_plugin="false", has_updated_plugin="false") + has_new_plugin="false", has_updated_plugin="false", + metadata_only="false") if outside_changes: actions.set_output("outside_files", "\n".join(outside_changes)) actions.log("No plugin changes detected - skipping plugin validation (author has write access).") diff --git a/.github/pluginctl/src/pluginctl/validate/plugin.py b/.github/pluginctl/src/pluginctl/validate/plugin.py index 328716a7..0b70a76d 100644 --- a/.github/pluginctl/src/pluginctl/validate/plugin.py +++ b/.github/pluginctl/src/pluginctl/validate/plugin.py @@ -19,12 +19,7 @@ from ..core import actions, gh, git from ..core.version import is_semver, is_dispatcharr_version, version_greater_than -from .detect import author_in_plugin_json, is_safe_name - -METADATA_ONLY_FIELDS = [ - "description", "repo_url", "discord_thread", "min_dispatcharr_version", - "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", -] +from .detect import METADATA_ONLY_FIELDS, author_in_plugin_json, changed_json_fields, is_safe_name _GH_DOWNLOAD_RE = re.compile(r"^https://github\.com/([^/]+)/([^/]+)/releases/download/([^/]+)/") @@ -311,7 +306,7 @@ def emit_fragment() -> None: if version_greater_than(version, old_version): rows.append(f"| Version bump | ✅ | `{old_version}` → `{version}` |") else: - changed_fields = _changed_fields(base_raw, raw) + changed_fields = changed_json_fields(base_raw, raw) metadata_only = all(f in METADATA_ONLY_FIELDS for f in changed_fields) if metadata_only and changed_fields: rows.append(f"| Version bump | ✅ | `{old_version}` (unchanged - metadata-only update) |") @@ -390,12 +385,6 @@ def emit_fragment() -> None: return 1 if failed else 0 -def _changed_fields(old: dict, new: dict) -> list[str]: - """Keys of NEW whose value differs from OLD (missing old -> null), sorted like jq keys.""" - changed = [k for k in new.keys() if old.get(k) != new[k]] - return sorted(changed) - - def _write(path: str, text: str) -> None: if path in ("/dev/stdout", "-"): print(text, end="") diff --git a/.github/pluginctl/tests/test_detect.py b/.github/pluginctl/tests/test_detect.py index d0e01075..21e4e0bf 100644 --- a/.github/pluginctl/tests/test_detect.py +++ b/.github/pluginctl/tests/test_detect.py @@ -83,3 +83,64 @@ def test_unsafe_named_companion_folder_blocks_pr(tmp_path, monkeypatch): assert outputs["close_pr"] == "true" assert outputs["close_reason"] == "unsafe-plugin-name" assert outputs["matrix"] == "[]" + + +def test_metadata_only_update_accepts_approved_field_removal(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo-plugin" + pdir.mkdir(parents=True) + (pdir / "plugin.json").write_text(json.dumps({"version": "1.0.0"}), encoding="utf-8") + monkeypatch.setattr(git, "show", lambda *_: json.dumps({ + "version": "1.0.0", "maintainers": ["alice"], + })) + + assert detect.is_metadata_only_update( + "demo-plugin", "main", ["plugins/demo-plugin/plugin.json"] + ) + + +@pytest.mark.parametrize( + ("changed_files", "current"), + [ + (["plugins/demo-plugin/main.py", "plugins/demo-plugin/plugin.json"], + {"version": "1.0.0", "description": "Updated"}), + (["plugins/demo-plugin/plugin.json"], + {"version": "1.0.1", "description": "Updated"}), + ], +) +def test_metadata_only_update_rejects_source_or_unapproved_changes(tmp_path, monkeypatch, + changed_files, current): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo-plugin" + pdir.mkdir(parents=True) + (pdir / "plugin.json").write_text(json.dumps(current), encoding="utf-8") + monkeypatch.setattr(git, "show", lambda *_: json.dumps({ + "version": "1.0.0", "description": "Original", + })) + + assert not detect.is_metadata_only_update("demo-plugin", "main", changed_files) + + +def test_detect_marks_only_an_all_metadata_matrix(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + out = tmp_path / "gh_output" + out.write_text("", encoding="utf-8") + monkeypatch.setenv("GITHUB_OUTPUT", str(out)) + pdir = tmp_path / "plugins" / "demo-plugin" + pdir.mkdir(parents=True) + (pdir / "plugin.json").write_text(json.dumps({ + "version": "1.0.0", "description": "Updated", "author": "alice", + }), encoding="utf-8") + + monkeypatch.setattr(git, "merge_base", lambda *a, **k: "base-sha") + monkeypatch.setattr(git, "diff_name_only", lambda *a, **k: [ + "plugins/demo-plugin/plugin.json", + ]) + monkeypatch.setattr(git, "object_exists", lambda *a, **k: True) + monkeypatch.setattr(git, "show", lambda *a, **k: json.dumps({ + "version": "1.0.0", "description": "Original", "author": "alice", + })) + monkeypatch.setattr(gh, "has_write_access", lambda *a, **k: True) + + assert detect.run("alice", "main", repo="org/repo") == 0 + assert _outputs(out)["metadata_only"] == "true" diff --git a/.github/workflows/_clamav-scan.yml b/.github/workflows/_clamav-scan.yml index 1207b95d..794ce970 100644 --- a/.github/workflows/_clamav-scan.yml +++ b/.github/workflows/_clamav-scan.yml @@ -16,6 +16,9 @@ on: base_ref: required: true type: string + metadata_only: + required: true + type: string outputs: clamav_status: value: ${{ jobs.scan.outputs.clamav_status }} @@ -120,7 +123,7 @@ jobs: plugin_json="plugins/$plugin_name/plugin.json" [[ ! -f "$plugin_json" ]] && continue source_type=$(jq -r '.source_type // "local"' "$plugin_json") - if [[ "$source_type" == "external" ]]; then + if [[ "$source_type" == "external" && "${{ inputs.metadata_only }}" != "true" ]]; then version=$(jq -r '.version' "$plugin_json") source_url_template=$(jq -r '.source_url // ""' "$plugin_json") source_url="${source_url_template//\{version\}/$version}" @@ -136,6 +139,9 @@ jobs: SCAN_TARGETS+=("plugins/$plugin_name") fi else + if [[ "$source_type" == "external" ]]; then + echo "Metadata-only update, scanning checked-out metadata for $plugin_name" + fi SCAN_TARGETS+=("plugins/$plugin_name") fi done < <(echo '${{ inputs.matrix }}' | jq -r '.[]') diff --git a/.github/workflows/validate-plugin.yml b/.github/workflows/validate-plugin.yml index 043cefd9..3fb607a9 100644 --- a/.github/workflows/validate-plugin.yml +++ b/.github/workflows/validate-plugin.yml @@ -85,6 +85,7 @@ jobs: pub_key_changed: ${{ steps.detect.outputs.pub_key_changed }} has_new_plugin: ${{ steps.detect.outputs.has_new_plugin }} has_updated_plugin: ${{ steps.detect.outputs.has_updated_plugin }} + metadata_only: ${{ steps.detect.outputs.metadata_only }} steps: - name: Trusted checkout uses: ./.github/actions/trusted-checkout @@ -186,7 +187,7 @@ jobs: # -------------------------------------------------------------------------- codeql-analyze: needs: [detect-changes, validate-plugin, validate-title] - if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.validate-title.outputs.title_valid == 'true' + if: needs.validate-plugin.result == 'success' && needs.detect-changes.outputs.skip_validation != 'true' && needs.detect-changes.outputs.metadata_only != 'true' && needs.validate-title.outputs.title_valid == 'true' uses: ./.github/workflows/_codeql-scan.yml with: matrix: ${{ needs.detect-changes.outputs.matrix }} @@ -204,6 +205,7 @@ jobs: matrix: ${{ needs.detect-changes.outputs.matrix }} pr_number: ${{ github.event.pull_request.number }} base_ref: ${{ github.event.pull_request.base.ref }} + metadata_only: ${{ needs.detect-changes.outputs.metadata_only }} # -------------------------------------------------------------------------- # Job 6b: Tooling test suite (reusable workflow) for authorized repo updates From 9c5c7e96d5bec5dcfc2e65a49f5f410a99476fa1 Mon Sep 17 00:00:00 2001 From: Seth Van Niekerk Date: Sat, 22 Aug 2026 19:41:01 -0400 Subject: [PATCH 13/13] feat(manifest): disclose AI-assisted plugins Publish ai_assisted metadata from manual plugin declarations or recognized source-history attribution. Add expanded agent detection, rollback cleanup, concise documentation, and coverage. --- .../pluginctl/src/pluginctl/feature_flags.py | 6 +- .../src/pluginctl/publish/ai_assisted.py | 72 +++++++++++++++++++ .../src/pluginctl/publish/manifest.py | 24 +++++-- .../src/pluginctl/validate/detect.py | 2 +- .github/pluginctl/tests/test_ai_assisted.py | 60 ++++++++++++++++ .github/pluginctl/tests/test_detect.py | 13 ++++ .github/pluginctl/tests/test_manifest.py | 47 ++++++++++++ .github/pluginctl/tests/test_validate.py | 25 +++++++ CONTRIBUTING.md | 7 ++ README.md | 6 ++ 10 files changed, 250 insertions(+), 12 deletions(-) create mode 100644 .github/pluginctl/src/pluginctl/publish/ai_assisted.py create mode 100644 .github/pluginctl/tests/test_ai_assisted.py diff --git a/.github/pluginctl/src/pluginctl/feature_flags.py b/.github/pluginctl/src/pluginctl/feature_flags.py index 1094d8c7..d4618f42 100644 --- a/.github/pluginctl/src/pluginctl/feature_flags.py +++ b/.github/pluginctl/src/pluginctl/feature_flags.py @@ -1,9 +1,7 @@ """Hardcoded rollout switches for trusted pluginctl automation.""" -# Flip only after the sandbox-bypass query pack has completed its observation period. -# This is deliberately not environment configurable, so PR code cannot enable it. SANDBOX_BYPASS_DETECTION = False -# Keep legacy root_url manifests until the Python publisher fully replaces the -# shell workflow. This is deliberately not environment configurable. SPLIT_MANIFEST_BASE_URLS = False + +AI_ASSISTED_MANIFEST_DISCLOSURE = True diff --git a/.github/pluginctl/src/pluginctl/publish/ai_assisted.py b/.github/pluginctl/src/pluginctl/publish/ai_assisted.py new file mode 100644 index 00000000..66f9348e --- /dev/null +++ b/.github/pluginctl/src/pluginctl/publish/ai_assisted.py @@ -0,0 +1,72 @@ +"""Best-effort detection of AI attribution in plugin Git history.""" + +from __future__ import annotations + +import re + +from ..core import actions, git + + +TRAILER_RE = re.compile( + r"^(?:co-authored-by|ai-assisted-by|assisted-by|generated-by):\s*(.+)$", + re.IGNORECASE | re.MULTILINE, +) + +# Match only agent names in authorship fields, never prose in a commit message. +AGENT_NAME_RE = re.compile( + r"^(?:" + r"claude (?:code|opus|sonnet|haiku|fable)(?: [0-9.]+)?|" + r"(?:openai )?codex|chatgpt|" + r"(?:github )?copilot|gemini(?: cli)?|cursor|aider|cline|roo code|continue|" + r"windsurf|codeium|devin|amp|(?:amazon )?q|tabnine|jetbrains ai|kiro|" + r"augment code|qodo|codiumai|sweep ai|replit agent|hermes agent" + r")(?: \([^)]*\))?$", + re.IGNORECASE, +) + +# Exact known identities are stronger evidence than a display name and cover +# formats already used in this repository. +AGENT_EMAILS = { + "noreply@anthropic.com", + "hermes-agent@g1tech.us", +} + + +def has_attribution(source_branch: str, plugin_name: str) -> bool: + """Return whether source-branch history attributes a plugin change to an AI agent.""" + proc = git.run( + "log", f"origin/{source_branch}", "--format=%aN%x1f%aE%x1f%B%x1e", + "--", f"plugins/{plugin_name}", check=False, + ) + if proc.returncode != 0: + actions.warning(f"Could not inspect AI attribution history for {plugin_name}; " + "using manual disclosure only.") + return False + + for record in proc.stdout.split("\x1e"): + if not record.strip(): + continue + try: + author_name, author_email, message = record.split("\x1f", 2) + except ValueError: + actions.warning(f"Could not parse AI attribution history for {plugin_name}; " + "using manual disclosure only.") + return False + if _is_agent_identity(author_name, author_email): + return True + for trailer in TRAILER_RE.findall(message): + name, email = _identity_parts(trailer) + if _is_agent_identity(name, email): + return True + return False + + +def _identity_parts(value: str) -> tuple[str, str]: + match = re.fullmatch(r"\s*(.*?)\s*<([^<>]+)>\s*", value) + if match: + return match.group(1), match.group(2) + return value.strip(), "" + + +def _is_agent_identity(name: str, email: str) -> bool: + return email.strip().lower() in AGENT_EMAILS or bool(AGENT_NAME_RE.fullmatch(name.strip())) diff --git a/.github/pluginctl/src/pluginctl/publish/manifest.py b/.github/pluginctl/src/pluginctl/publish/manifest.py index 948d2b2a..a8bac4cd 100644 --- a/.github/pluginctl/src/pluginctl/publish/manifest.py +++ b/.github/pluginctl/src/pluginctl/publish/manifest.py @@ -22,6 +22,7 @@ from ..core.jsonio import drop_none from ..core.timeutil import now_iso from ..core.version import versioned_tags +from . import ai_assisted def _metadata_fields(metadata: dict, min_da, max_da) -> dict: @@ -79,7 +80,7 @@ def override_current_min_max(versioned_zips: list[dict], current_version: str, def build_plugin_entry(plugin_raw: dict, plugin_name: str, latest_url: str, registry_url: str, registry_name: str, versioned_zips: list[dict], latest_metadata: dict, - latest_size_kb) -> dict: + latest_size_kb, ai_assisted_disclosed: bool = False) -> dict: """Per-plugin manifest ``.manifest`` payload (metadata//manifest.json).""" def g(key): return plugin_raw.get(key) @@ -102,6 +103,7 @@ def g(key): "author": g("author"), "maintainers": g("maintainers"), "license": g("license"), + "ai_assisted": True if ai_assisted_disclosed else None, "deprecated": True if g("deprecated") is True else None, "source_type": "external" if g("source_type") == "external" else None, "source_url": g("source_url"), @@ -121,8 +123,9 @@ def trim_description(desc: str) -> str: def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, - latest_size_kb, min_da: str, max_da: str, - latest_url: str, manifest_url: str) -> dict: + latest_size_kb, min_da: str, max_da: str, + latest_url: str, manifest_url: str, + ai_assisted_disclosed: bool = False) -> dict: """Compact entry for the root manifest's ``plugins[]``.""" license_id = plugin_raw.get("license") or "" return drop_none({ @@ -132,6 +135,7 @@ def build_root_entry(plugin_raw: dict, plugin_name: str, latest_metadata: dict, "manifest_url": manifest_url, "author": plugin_raw.get("author") or "", "license": license_id if license_id != "" else None, + "ai_assisted": True if ai_assisted_disclosed else None, "deprecated": True if plugin_raw.get("deprecated") is True else None, "last_updated": latest_metadata.get("last_updated") if latest_metadata else None, "latest_version": latest_metadata.get("version") if latest_metadata else None, @@ -342,6 +346,11 @@ def sign_if_needed(path: str, written: bool) -> None: min_da = str(raw.get("min_dispatcharr_version") or "") max_da = str(raw.get("max_dispatcharr_version") or "") unlisted = raw.get("unlisted") is True + ai_assisted_disclosed = ( + feature_flags.AI_ASSISTED_MANIFEST_DISCLOSURE and + (raw.get("ai_assisted") is True or + ai_assisted.has_attribution(source_branch, plugin_name)) + ) actions.log(f" {plugin_name}") existing_manifest_file = f"metadata/{plugin_name}/manifest.json" @@ -400,8 +409,8 @@ def sign_if_needed(path: str, written: bool) -> None: versioned_zips = override_current_min_max(versioned_zips, current_version, min_da, max_da) plugin_entry = build_plugin_entry(raw, plugin_name, latest_url, registry_url, - registry_name, versioned_zips, latest_metadata, - latest_size_kb) + registry_name, versioned_zips, latest_metadata, + latest_size_kb, ai_assisted_disclosed) written = write_manifest_if_changed(existing_manifest_file, plugin_entry, generated_at) sign_if_needed(existing_manifest_file, written) @@ -412,8 +421,9 @@ def sign_if_needed(path: str, written: bool) -> None: if feature_flags.SPLIT_MANIFEST_BASE_URLS else f"{raw_releases_url}/metadata/{plugin_name}/manifest.json") root_entries.append(build_root_entry(raw, plugin_name, latest_metadata, - latest_size_kb, min_da, max_da, - latest_url, manifest_url)) + latest_size_kb, min_da, max_da, + latest_url, manifest_url, + ai_assisted_disclosed)) plugin_count += 1 inner_root = build_root_manifest( diff --git a/.github/pluginctl/src/pluginctl/validate/detect.py b/.github/pluginctl/src/pluginctl/validate/detect.py index da814343..053c5fcc 100644 --- a/.github/pluginctl/src/pluginctl/validate/detect.py +++ b/.github/pluginctl/src/pluginctl/validate/detect.py @@ -19,7 +19,7 @@ PUB_KEY_PATH = ".github/scripts/keys/dispatcharr-plugins.pub" METADATA_ONLY_FIELDS = { "description", "repo_url", "discord_thread", "min_dispatcharr_version", - "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", + "max_dispatcharr_version", "deprecated", "unlisted", "maintainers", "ai_assisted", } diff --git a/.github/pluginctl/tests/test_ai_assisted.py b/.github/pluginctl/tests/test_ai_assisted.py new file mode 100644 index 00000000..656e86cd --- /dev/null +++ b/.github/pluginctl/tests/test_ai_assisted.py @@ -0,0 +1,60 @@ +import subprocess + +import pytest + +from pluginctl.core import git +from pluginctl.publish import ai_assisted + + +def _log(author_name="Alice", author_email="alice@example.com", message="Update plugin"): + return "\x1f".join((author_name, author_email, message)) + "\x1e" + + +def _stub_log(monkeypatch, output, returncode=0): + monkeypatch.setattr( + git, "run", lambda *args, **kwargs: subprocess.CompletedProcess(args, returncode, output, ""), + ) + + +@pytest.mark.parametrize("identity", [ + "Claude Opus 5 (1M context)", "Claude Code", "OpenAI Codex", "Codex", "ChatGPT", + "GitHub Copilot", "Gemini CLI", "Cursor", "Aider", "Cline", "Roo Code", "Continue", + "Windsurf", "Codeium", "Devin", "Amp", "Amazon Q", "Tabnine", "JetBrains AI", "Kiro", + "Augment Code", "Qodo", "CodiumAI", "Sweep AI", "Replit Agent", "Hermes Agent", +]) +def test_known_agent_names_are_recognized(identity): + assert ai_assisted._is_agent_identity(identity, "") + + +def test_anthropic_coauthor_trailer_is_recognized(monkeypatch): + _stub_log(monkeypatch, _log(message="Update\n\nCo-authored-by: Claude Opus 5 ")) + assert ai_assisted.has_attribution("main", "demo-plugin") + + +def test_hermes_trailer_and_nonstandard_key_are_recognized(monkeypatch): + _stub_log(monkeypatch, _log(message="Generated-by: Hermes Agent ")) + assert ai_assisted.has_attribution("main", "demo-plugin") + + +def test_agent_primary_author_is_recognized(monkeypatch): + _stub_log(monkeypatch, _log("Gemini CLI", "bot@example.com")) + assert ai_assisted.has_attribution("main", "demo-plugin") + + +def test_case_insensitive_trailer_is_recognized(monkeypatch): + _stub_log(monkeypatch, _log(message="AI-ASSISTED-BY: github copilot ")) + assert ai_assisted.has_attribution("main", "demo-plugin") + + +def test_human_coauthor_and_agent_prose_are_not_recognized(monkeypatch): + _stub_log(monkeypatch, _log(message=( + "This plugin uses an AI agent.\n\nCo-authored-by: Alice " + ))) + assert not ai_assisted.has_attribution("main", "demo-plugin") + + +def test_no_match_and_git_failure_return_false(monkeypatch): + _stub_log(monkeypatch, _log()) + assert not ai_assisted.has_attribution("main", "demo-plugin") + _stub_log(monkeypatch, "", returncode=1) + assert not ai_assisted.has_attribution("main", "demo-plugin") diff --git a/.github/pluginctl/tests/test_detect.py b/.github/pluginctl/tests/test_detect.py index 21e4e0bf..0029e6a7 100644 --- a/.github/pluginctl/tests/test_detect.py +++ b/.github/pluginctl/tests/test_detect.py @@ -99,6 +99,19 @@ def test_metadata_only_update_accepts_approved_field_removal(tmp_path, monkeypat ) +def test_metadata_only_update_accepts_ai_assisted_disclosure(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + pdir = tmp_path / "plugins" / "demo-plugin" + pdir.mkdir(parents=True) + (pdir / "plugin.json").write_text( + json.dumps({"version": "1.0.0", "ai_assisted": True}), encoding="utf-8") + monkeypatch.setattr(git, "show", lambda *_: json.dumps({"version": "1.0.0"})) + + assert detect.is_metadata_only_update( + "demo-plugin", "main", ["plugins/demo-plugin/plugin.json"] + ) + + @pytest.mark.parametrize( ("changed_files", "current"), [ diff --git a/.github/pluginctl/tests/test_manifest.py b/.github/pluginctl/tests/test_manifest.py index d3dd236c..e8c2fa76 100644 --- a/.github/pluginctl/tests/test_manifest.py +++ b/.github/pluginctl/tests/test_manifest.py @@ -73,6 +73,17 @@ def test_plugin_entry_key_order(): ] +def test_ai_assisted_is_emitted_only_for_positive_disclosure(): + zips = [m.build_version_entry(META, "u", 10, "1.2.0")] + plugin = m.build_plugin_entry(PLUGIN_RAW, "demo", "u", "registry", "registry", zips, META, 10, True) + root = m.build_root_entry(PLUGIN_RAW, "demo", META, 10, "", "", "u", "manifest", True) + + assert plugin["ai_assisted"] is True + assert root["ai_assisted"] is True + assert list(plugin).index("ai_assisted") == list(plugin).index("license") + 1 + assert list(root).index("ai_assisted") == list(root).index("license") + 1 + + def test_root_entry_key_order(): entry = m.build_root_entry( PLUGIN_RAW, "demo", META, 10, "v0.9.0", "", @@ -192,3 +203,39 @@ def test_generate_uses_split_manifest_and_copies_icon_when_enabled(tmp_path, mon assert "root_url" not in root assert root["plugins"][0]["manifest_url"] == "metadata/demo/manifest.json" assert (tmp_path / "metadata" / "demo" / "logo.png").read_bytes() == b"png" + + +def test_generate_emits_and_strips_ai_assisted_with_feature_flag(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + plugin_dir = tmp_path / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / "plugin.json").write_text(json.dumps({ + "name": "Demo", "version": "1.0.0", "description": "d", "ai_assisted": True, + }), encoding="utf-8") + monkeypatch.setattr(gh, "release_list_tags", lambda repo: []) + monkeypatch.setattr(feature_flags, "AI_ASSISTED_MANIFEST_DISCLOSURE", True) + + assert m.generate("main", "releases", "org/repo", "") == 0 + plugin_manifest = tmp_path / "metadata" / "demo" / "manifest.json" + assert json.loads(plugin_manifest.read_text(encoding="utf-8"))["manifest"]["ai_assisted"] is True + assert json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))["manifest"]["plugins"][0]["ai_assisted"] is True + + monkeypatch.setattr(feature_flags, "AI_ASSISTED_MANIFEST_DISCLOSURE", False) + assert m.generate("main", "releases", "org/repo", "") == 0 + assert "ai_assisted" not in json.loads(plugin_manifest.read_text(encoding="utf-8"))["manifest"] + assert "ai_assisted" not in json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))["manifest"]["plugins"][0] + + +def test_generate_emits_ai_assisted_for_history_evidence(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + plugin_dir = tmp_path / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / "plugin.json").write_text( + json.dumps({"name": "Demo", "version": "1.0.0", "description": "d"}), encoding="utf-8") + monkeypatch.setattr(gh, "release_list_tags", lambda repo: []) + monkeypatch.setattr(feature_flags, "AI_ASSISTED_MANIFEST_DISCLOSURE", True) + monkeypatch.setattr(m.ai_assisted, "has_attribution", lambda branch, plugin: True) + + assert m.generate("main", "releases", "org/repo", "") == 0 + root = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8"))["manifest"] + assert root["plugins"][0]["ai_assisted"] is True diff --git a/.github/pluginctl/tests/test_validate.py b/.github/pluginctl/tests/test_validate.py index 6a0bba73..0b2146d1 100644 --- a/.github/pluginctl/tests/test_validate.py +++ b/.github/pluginctl/tests/test_validate.py @@ -130,3 +130,28 @@ def test_description_newline_cannot_forge_meta_row(plugin_repo): f_repo, f_discord = fields[5], fields[6] assert "evil.example" not in f_repo assert "discord.gg/evil" not in f_discord + + +def test_ai_assisted_true_is_accepted(plugin_repo): + tmp_path, out = plugin_repo + pj = tmp_path / "plugins" / "demo-plugin" / "plugin.json" + data = json.loads(pj.read_text()) + data["ai_assisted"] = True + pj.write_text(json.dumps(data), encoding="utf-8") + + assert validate.run("demo-plugin", "alice", "main", str(tmp_path / "frag.md"), repo="org/repo") == 0 + assert _outputs(out)["result"] == "pass" + + +@pytest.mark.parametrize("value", [False, None, "true", 1]) +def test_ai_assisted_non_positive_values_are_ignored(plugin_repo, value): + tmp_path, out = plugin_repo + pj = tmp_path / "plugins" / "demo-plugin" / "plugin.json" + data = json.loads(pj.read_text()) + data["ai_assisted"] = value + pj.write_text(json.dumps(data), encoding="utf-8") + + fragment = tmp_path / "frag.md" + assert validate.run("demo-plugin", "alice", "main", str(fragment), repo="org/repo") == 0 + assert _outputs(out)["result"] == "pass" + assert "ai_assisted" not in fragment.read_text(encoding="utf-8") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1de0be4..16209567 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,6 +110,7 @@ At least one of `author` or `maintainers` must include your GitHub username. `au | `discord_thread` | `string` | URL to the associated Discord thread (must start with `http://` or `https://`) | | `deprecated` | `boolean` | Marks the plugin as deprecated. Default: `false` | | `unlisted` | `boolean` | Excludes the plugin from the root `manifest.json` (and the releases README) but still generates a per-plugin manifest. Default: `false` | +| `ai_assisted` | `boolean` | Set to `true` to voluntarily disclose AI assistance with this plugin. | | `source_type` | `string` | Set to `"external"` to declare this as an external plugin. Omit (or `"local"`) for standard plugins | | `source_url` | `string` | Required when `source_type` is `"external"`. Must be a GitHub Releases URL containing a `{version}` placeholder, e.g. `https://github.com/owner/repo/releases/download/v{version}/plugin.zip` | @@ -122,6 +123,7 @@ At least one of `author` or `maintainers` must include your GitHub username. `au "description": "Does something useful for Dispatcharr", "author": "your-github-username", "license": "MIT", + "ai_assisted": true, "source_type": "external", "source_url": "https://github.com/your-github-username/my-plugin/releases/download/v{version}/my-plugin.zip", "repo_url": "https://github.com/your-github-username/my-plugin", @@ -212,9 +214,14 @@ Version increments are enforced by the validation workflow. You cannot submit a - `max_dispatcharr_version` - `deprecated` - `unlisted` +- `ai_assisted` All other fields - including `name`, `author`, `license`, `source_url`, `source_type`, and any code changes - require a version bump. +`ai_assisted` may be voluntarily disclosed by a plugin author or detected from +recognized AI attribution in registry history. Omission does not assert that AI +was not used. + > **Changing the license?** A version bump is required because the license you publish under is binding for that release. Users who installed the previous version hold rights under the old license and those cannot be revoked. The new version carries the new license going forward. ## Licensing diff --git a/README.md b/README.md index c2adfd16..305788ba 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ The root `manifest.json` uses a `root_url` plus relative paths for ZIP downloads "slug": "my-plugin", "name": "My Plugin", "manifest_url": "https://raw.githubusercontent.com/Dispatcharr/Plugins/releases/metadata/my-plugin/manifest.json", + "ai_assisted": true, "latest_url": "my-plugin-latest/my-plugin-latest.zip", ... } @@ -89,6 +90,11 @@ To resolve a full ZIP download URL: `root_url + "/" + latest_url`. The `slug` matches the plugin folder name and can be used to construct other paths (e.g. icon: `plugins//logo.png` on the source branch). +`ai_assisted` may be voluntarily disclosed by a plugin author or detected from +recognized AI attribution in registry history. When present, the same boolean +appears in the per-plugin manifest. Omission does not assert that AI was not +used. + ## Verifying Manifest Signatures Each manifest file embeds its GPG signature directly. The `signature` field covers the compact (`jq -c '.manifest'`) form of the `manifest` payload.