diff --git a/README.md b/README.md index fa2b909..b2950e0 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,15 @@ Status: in progress. The design is in `specs/next-ci-reliability` of `--until 2026-10-14T11:00` ends the period in the past, so a baseline can be reproduced after later changes; `--limit` raises the number of runs read per repository (default 300). +- `standards/audit.py` (available): audits every repository of the organisation against + `standards/standards.json` and prints each deviation as `error`, `warning` or `info`: branch + deletion after merge, Dependabot, CODEOWNERS, secret scanning and a recognised license on public + repositories, a CI that calls the shared gate pinned by a full SHA with a `# vX.Y.Z` comment, too + many leftover branches, and the organisation's two-factor requirement. An exception names a + repository and a rule, gives a reason and expires; an expired one is itself an error. + `py -3 standards/audit.py [--fail-on error]`, or `report/ci_report.py --audit` to append it to the + weekly report. + ## Rules - No secrets, no business logic and no repository-specific values: everything comes in as an input. diff --git a/report/ci_report.py b/report/ci_report.py index 93b2019..9a3d112 100644 --- a/report/ci_report.py +++ b/report/ci_report.py @@ -17,6 +17,7 @@ import argparse import datetime as dt +import importlib.util import json import statistics import subprocess @@ -305,6 +306,24 @@ def render( return "\n".join(lines) +def load_audit(): + """The standards audit module, found beside this report (`../standards/audit.py`).""" + path = Path(__file__).resolve().parent.parent / "standards" / "audit.py" + spec = importlib.util.spec_from_file_location("standards_audit", path) + module = importlib.util.module_from_spec(spec) + sys.modules["standards_audit"] = module # dataclasses look the module up by name + spec.loader.exec_module(module) + return module + + +def audit_section(audit_text: str) -> str: + """The audit report as a section of this one: its title becomes a second-level heading.""" + lines = audit_text.split("\n") + if lines and lines[0] == "# Standards audit": + lines[0] = "## Standards audit" + return "\n".join(lines) + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( @@ -324,6 +343,11 @@ def main(argv: list[str] | None = None) -> int: default=300, help="runs read per repository (gh allows up to 1000)", ) + parser.add_argument( + "--audit", + action="store_true", + help="append the standards audit of the organisation", + ) parser.add_argument("--output", help="write the report here instead of printing it") args = parser.parse_args(argv) @@ -343,6 +367,12 @@ def main(argv: list[str] | None = None) -> int: for repo in repos ] text = render(stats, args.days, now, until=until) + if args.audit: + audit = load_audit() + standard_path = Path(audit.__file__).with_name("standards.json") + text += "\n\n" + audit_section( + audit.render(audit.run(standard_path, now=now), now) + ) if args.output: Path(args.output).write_text(text + "\n", encoding="utf-8") else: diff --git a/standards/audit.py b/standards/audit.py new file mode 100644 index 0000000..6474ceb --- /dev/null +++ b/standards/audit.py @@ -0,0 +1,376 @@ +"""Audit the repositories of an organisation against a written standard. + + py -3 audit.py # print the deviations + py -3 audit.py --fail-on error # exit 1 when there is an error (for a CI step) + +The standard is `standards.json` beside this script: thresholds, the severity of each rule and the +exceptions. An exception names a repository and a rule, says why, and expires; an expired exception +stops silencing the rule and is itself an error, so "on purpose" cannot turn into "forgotten". +Severities: `error` breaks a guarantee, `warning` is hygiene, `info` is an observation. + +Rules (per repository unless noted): delete-branch-on-merge, dependabot-config, codeowners, +secret-scanning (public repositories, where it is free), license (public repositories), +has-ci, shared-gate, gate-pin (the shared gate is pinned by a full commit SHA with a vX.Y.Z comment, +which is what online zizmor verifies), topic-branches, and for the organisation org-2fa and +org-2fa-method. Archived repositories are skipped. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import re +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +SEVERITIES = ("error", "warning", "info") +GATE_REFERENCE = "naindev-contrast3d/ci-shared/.github/workflows/" +GOOD_PIN = re.compile(r"@[0-9a-f]{40}\s+#\s+v\d+\.\d+\.\d+\b") + +DEFAULT_SEVERITY = { + "delete-branch-on-merge": "error", + "dependabot-config": "error", + "license": "error", + "gate-pin": "error", + "org-2fa": "error", + "exception-expired": "error", + "codeowners": "warning", + "secret-scanning": "warning", + "has-ci": "warning", + "shared-gate": "warning", + "topic-branches": "warning", + "org-2fa-method": "warning", +} + + +class StandardError(ValueError): + """The standard file cannot be used.""" + + +@dataclass(frozen=True) +class Finding: + repo: str + rule: str + severity: str + message: str + + +def run_gh(command: list[str]) -> str: + # Every command is built in this file from fixed words and names GitHub returned. + no_window = getattr(subprocess, "CREATE_NO_WINDOW", 0) + return subprocess.run( + command, + check=True, + capture_output=True, + text=True, + encoding="utf-8", + creationflags=no_window, + ).stdout + + +class Github: + """The reads the audit needs, over the `gh` command line.""" + + def __init__(self, run: Callable[[list[str]], str] = run_gh): + self._run = run + + def org(self, org: str) -> dict: + return json.loads(self._run(["gh", "api", f"orgs/{org}"]) or "{}") + + def org_repos(self, org: str) -> list[dict]: + out = self._run( + [ + "gh", + "api", + f"orgs/{org}/repos?per_page=100&type=all", + "--paginate", + "--slurp", + ] + ) + listed = json.loads(out or "[]") + if listed and isinstance(listed[0], list): # --slurp returns one array per page + listed = [item for page in listed for item in page] + repos = [] + for entry in listed: + # The list endpoint omits settings such as delete_branch_on_merge: read each one. + detail = json.loads( + self._run(["gh", "api", f"repos/{org}/{entry['name']}"]) or "{}" + ) + repos.append(detail if isinstance(detail, dict) and detail else entry) + return repos + + def file_text(self, full_name: str, path: str) -> str | None: + try: + return self._run( + [ + "gh", + "api", + "-H", + "Accept: application/vnd.github.raw+json", + f"repos/{full_name}/contents/{path}", + ] + ) + except subprocess.CalledProcessError: + return None + + def dir_names(self, full_name: str, path: str) -> list[str]: + try: + out = self._run(["gh", "api", f"repos/{full_name}/contents/{path}"]) + except subprocess.CalledProcessError: + return [] + listing = json.loads(out or "[]") + return [item["name"] for item in listing] if isinstance(listing, list) else [] + + def branches(self, full_name: str) -> list[str]: + out = self._run( + [ + "gh", + "api", + f"repos/{full_name}/branches?per_page=100", + "--paginate", + "--jq", + ".[].name", + ] + ) + return [line.strip() for line in out.splitlines() if line.strip()] + + def insecure_two_factor_members(self, org: str) -> list[str]: + try: + out = self._run(["gh", "api", f"orgs/{org}/members?filter=2fa_insecure"]) + except subprocess.CalledProcessError: + return [] # needs organisation owner rights + return [member["login"] for member in json.loads(out or "[]")] + + +def validate_standard(standard: dict) -> None: + for rule, severity in standard.get("severity", {}).items(): + if severity not in SEVERITIES: + raise StandardError( + f"severity of {rule} must be one of {', '.join(SEVERITIES)}" + ) + for item in standard.get("exceptions", []): + for key in ("repo", "rule", "reason", "expires"): + if not item.get(key): + raise StandardError(f"every exception needs `{key}`: {item}") + try: + dt.date.fromisoformat(item["expires"]) + except ValueError as exc: + raise StandardError( + f"`expires` must be a date (YYYY-MM-DD): {item}" + ) from exc + + +def audit(gh, standard: dict, now: dt.datetime) -> list[Finding]: + validate_standard(standard) + org = standard["organisation"] + today = now.date() + severity = {**DEFAULT_SEVERITY, **standard.get("severity", {})} + exceptions = standard.get("exceptions", []) + active = { + (item["repo"], item["rule"]) + for item in exceptions + if dt.date.fromisoformat(item["expires"]) >= today + } + findings: list[Finding] = [] + + def add(repo: str, rule: str, message: str, level: str | None = None) -> None: + if (repo, rule) in active: + return + findings.append(Finding(repo, rule, level or severity[rule], message)) + + for item in exceptions: + if dt.date.fromisoformat(item["expires"]) < today: + findings.append( + Finding( + item["repo"], + "exception-expired", + severity["exception-expired"], + f"the exception for `{item['rule']}` expired on {item['expires']} " + f"({item['reason']}); decide again", + ) + ) + + info = gh.org(org) + if not info.get("two_factor_requirement_enabled"): + add( + org, + "org-2fa", + "the organisation does not require two-factor authentication", + ) + insecure = gh.insecure_two_factor_members(org) + if insecure: + add( + org, + "org-2fa-method", + "members with an insecure second factor: " + ", ".join(insecure), + ) + + for repo in gh.org_repos(org): + if repo.get("archived"): + continue + full = repo["full_name"] + public = not repo.get("private", True) + _audit_repo(gh, standard, repo, full, public, add) + return findings + + +def _audit_repo(gh, standard: dict, repo: dict, full: str, public: bool, add) -> None: + if not repo.get("delete_branch_on_merge"): + add( + full, + "delete-branch-on-merge", + "merged branches are not deleted automatically", + ) + + workflow_names = [ + name + for name in gh.dir_names(full, ".github/workflows") + if name.endswith((".yml", ".yaml")) + ] + if not workflow_names: + add(full, "has-ci", "no GitHub Actions workflow") + + has_dependabot = any( + gh.file_text(full, f".github/dependabot.{ext}") is not None + for ext in ("yml", "yaml") + ) + if not has_dependabot: + add( + full, + "dependabot-config", + "no .github/dependabot.yml", + "error" if workflow_names else "warning", + ) + + if not any( + gh.file_text(full, path) is not None + for path in (".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS") + ): + add(full, "codeowners", "no CODEOWNERS file") + + if public: + security = repo.get("security_and_analysis") or {} + scanning = (security.get("secret_scanning") or {}).get("status") == "enabled" + protection = (security.get("secret_scanning_push_protection") or {}).get( + "status" + ) == "enabled" + if not (scanning and protection): + add( + full, + "secret-scanning", + "secret scanning or push protection is off (free on public repositories)", + ) + spdx = (repo.get("license") or {}).get("spdx_id") + if spdx in (None, "NOASSERTION"): + add(full, "license", "GitHub does not recognise the license") + + _audit_gate(gh, workflow_names, full, add) + + limit = standard.get("max_topic_branches", 8) + topics = [b for b in gh.branches(full) if b != repo.get("default_branch", "main")] + if len(topics) > limit: + add( + full, + "topic-branches", + f"{len(topics)} branches besides the default one (limit {limit})", + ) + + +def _audit_gate(gh, workflow_names: list[str], full: str, add) -> None: + if not workflow_names: + return + references = [] + for name in workflow_names: + text = gh.file_text(full, f".github/workflows/{name}") or "" + references += [ + line + for line in text.splitlines() + if GATE_REFERENCE in line + and "uses:" in line + and not line.lstrip().startswith("#") + ] + if not references: + add(full, "shared-gate", "no workflow calls the shared gate of ci-shared") + return + if not all(GOOD_PIN.search(line) for line in references): + add( + full, + "gate-pin", + "the shared workflow must be pinned by a full commit SHA with a `# vX.Y.Z` comment", + ) + + +def exit_code(findings: list[Finding], fail_on: str | None) -> int: + if fail_on is None: + return 0 + levels = {"error": ("error",), "warning": ("error", "warning")}[fail_on] + return 1 if any(f.severity in levels for f in findings) else 0 + + +def render(findings: list[Finding], now: dt.datetime) -> str: + counts = { + level: sum(1 for f in findings if f.severity == level) for level in SEVERITIES + } + parts = [f"{counts['error']} error{'s' if counts['error'] != 1 else ''}"] + parts.append(f"{counts['warning']} warning{'s' if counts['warning'] != 1 else ''}") + if counts["info"]: + parts.append(f"{counts['info']} info") + lines = [ + "# Standards audit", + "", + f"{now.strftime('%Y-%m-%d')}: {', '.join(parts)}.", + "", + ] + if not findings: + lines.append("No deviations from the standard.") + return "\n".join(lines) + order = {level: index for index, level in enumerate(SEVERITIES)} + for repo in sorted({f.repo for f in findings}): + lines.append(f"### {repo}") + for f in sorted( + (x for x in findings if x.repo == repo), + key=lambda x: (order[x.severity], x.rule), + ): + lines.append(f"- {f.severity} `{f.rule}`: {f.message}") + lines.append("") + return "\n".join(lines) + + +def run( + standard_path: Path | str, gh=None, now: dt.datetime | None = None +) -> list[Finding]: + standard = json.loads(Path(standard_path).read_text(encoding="utf-8")) + return audit(gh or Github(), standard, now or dt.datetime.now(dt.UTC)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--standard", default=str(Path(__file__).with_name("standards.json")) + ) + parser.add_argument( + "--fail-on", choices=("error", "warning"), help="exit 1 from this severity" + ) + parser.add_argument("--output", help="write the report here instead of printing it") + args = parser.parse_args(argv) + + now = dt.datetime.now(dt.UTC) + try: + findings = run(args.standard, now=now) + except StandardError as exc: + print(f"standard error: {exc}", file=sys.stderr) + return 2 + text = render(findings, now) + if args.output: + Path(args.output).write_text(text + "\n", encoding="utf-8") + else: + print(text) + return exit_code(findings, args.fail_on) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/standards/standards.json b/standards/standards.json new file mode 100644 index 0000000..bc42174 --- /dev/null +++ b/standards/standards.json @@ -0,0 +1,43 @@ +{ + "organisation": "naindev-contrast3d", + "max_topic_branches": 8, + "severity": {}, + "exceptions": [ + { + "repo": "naindev-contrast3d/contrast-hub", + "rule": "shared-gate", + "reason": "its own `sello` job is the aggregate check and publishes the ci-ok seal (spec 365, ADR-0092)", + "expires": "2027-04-01" + }, + { + "repo": "naindev-contrast3d/ci-shared", + "rule": "shared-gate", + "reason": "it is the source of the gate and calls it by a local path in its own CI", + "expires": "2027-04-01" + }, + { + "repo": "naindev-contrast3d/.github", + "rule": "has-ci", + "reason": "the organisation profile repository holds only a README and community files", + "expires": "2027-04-01" + }, + { + "repo": "naindev-contrast3d/.github", + "rule": "dependabot-config", + "reason": "no workflows and no dependencies to update", + "expires": "2027-04-01" + }, + { + "repo": "naindev-contrast3d/.github", + "rule": "license", + "reason": "the organisation profile repository is documentation, not reusable code", + "expires": "2027-04-01" + }, + { + "repo": "naindev-contrast3d/dse", + "rule": "shared-gate", + "reason": "the session 'dse github move' is building its own CI; adoption is T23 of next-ci-reliability", + "expires": "2026-11-15" + } + ] +} diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..a32866a --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,380 @@ +"""Tests for standards/audit.py: each rule, the exceptions, the severities and the rendering.""" + +from __future__ import annotations + +import datetime as dt +import json +import subprocess +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "standards")) + +import audit + +ORG = "naindev-contrast3d" +NOW = dt.datetime(2026, 10, 14, 12, 0, 0, tzinfo=dt.UTC) +GOOD_PIN = ( + " uses: naindev-contrast3d/ci-shared/.github/workflows/gate.yml" + "@150de5008682330747ba6880f1649b388858c206 # v0.1.0\n" +) + + +def repo( + name, *, private=True, delete_branch=True, spdx="MIT", scanning=True, archived=False +): + status = "enabled" if scanning else "disabled" + return { + "name": name, + "full_name": f"{ORG}/{name}", + "private": private, + "visibility": "private" if private else "public", + "archived": archived, + "default_branch": "main", + "delete_branch_on_merge": delete_branch, + "allow_merge_commit": True, + "allow_squash_merge": True, + "allow_rebase_merge": True, + "license": {"spdx_id": spdx} if spdx else None, + "security_and_analysis": { + "secret_scanning": {"status": status}, + "secret_scanning_push_protection": {"status": status}, + }, + } + + +class FakeGithub: + def __init__( + self, repos, files=None, dirs=None, branches=None, two_fa=True, insecure=() + ): + self._repos = repos + self._files = files or {} + self._dirs = dirs or {} + self._branches = branches or {} + self._two_fa = two_fa + self._insecure = list(insecure) + + def org(self, org): + return {"two_factor_requirement_enabled": self._two_fa} + + def org_repos(self, org): + return self._repos + + def file_text(self, full_name, path): + return self._files.get((full_name, path)) + + def dir_names(self, full_name, path): + return self._dirs.get((full_name, path), []) + + def branches(self, full_name): + return self._branches.get(full_name, ["main"]) + + def insecure_two_factor_members(self, org): + return self._insecure + + +def good_files(name): + full = f"{ORG}/{name}" + return ( + { + (full, ".github/dependabot.yml"): "version: 2\n", + (full, ".github/CODEOWNERS"): "* @Nain9Dev\n", + (full, ".github/workflows/ci.yml"): "jobs:\n gate:\n" + GOOD_PIN, + }, + {(full, ".github/workflows"): ["ci.yml"]}, + ) + + +def standard(**overrides): + base = { + "organisation": ORG, + "max_topic_branches": 8, + "severity": {}, + "exceptions": [], + } + base.update(overrides) + return base + + +def clean_world(**kw): + files, dirs = good_files("app") + return FakeGithub([repo("app")], files, dirs, **kw) + + +def rules(findings, name="app"): + return {f.rule for f in findings if f.repo == f"{ORG}/{name}"} + + +class Rules(unittest.TestCase): + def test_a_repository_that_follows_the_standard_has_no_findings(self): + self.assertEqual(audit.audit(clean_world(), standard(), NOW), []) + + def test_branch_deletion_after_merge_must_be_on(self): + files, dirs = good_files("app") + gh = FakeGithub([repo("app", delete_branch=False)], files, dirs) + findings = audit.audit(gh, standard(), NOW) + self.assertEqual(rules(findings), {"delete-branch-on-merge"}) + self.assertEqual(findings[0].severity, "error") + + def test_dependabot_is_an_error_when_the_repository_has_workflows(self): + files, dirs = good_files("app") + del files[(f"{ORG}/app", ".github/dependabot.yml")] + findings = audit.audit(FakeGithub([repo("app")], files, dirs), standard(), NOW) + self.assertEqual(rules(findings), {"dependabot-config"}) + self.assertEqual(findings[0].severity, "error") + + def test_codeowners_is_a_warning_and_any_of_the_three_places_counts(self): + files, dirs = good_files("app") + del files[(f"{ORG}/app", ".github/CODEOWNERS")] + findings = audit.audit(FakeGithub([repo("app")], files, dirs), standard(), NOW) + self.assertEqual( + (rules(findings), findings[0].severity), ({"codeowners"}, "warning") + ) + files[(f"{ORG}/app", "docs/CODEOWNERS")] = "* @x\n" + self.assertEqual( + audit.audit(FakeGithub([repo("app")], files, dirs), standard(), NOW), [] + ) + + def test_a_public_repository_must_have_secret_scanning_and_push_protection(self): + files, dirs = good_files("pub") + gh = FakeGithub([repo("pub", private=False, scanning=False)], files, dirs) + findings = audit.audit(gh, standard(), NOW) + self.assertEqual( + (rules(findings, "pub"), findings[0].severity), + ({"secret-scanning"}, "warning"), + ) + + def test_a_private_repository_is_not_asked_for_paid_secret_scanning(self): + files, dirs = good_files("app") + gh = FakeGithub([repo("app", private=True, scanning=False)], files, dirs) + self.assertEqual(audit.audit(gh, standard(), NOW), []) + + def test_a_public_repository_needs_a_license_github_recognises(self): + files, dirs = good_files("pub") + for spdx in ("NOASSERTION", None): + with self.subTest(spdx=spdx): + gh = FakeGithub([repo("pub", private=False, spdx=spdx)], files, dirs) + findings = audit.audit(gh, standard(), NOW) + self.assertEqual( + (rules(findings, "pub"), findings[0].severity), + ({"license"}, "error"), + ) + + def test_a_private_repository_may_have_no_license(self): + files, dirs = good_files("app") + gh = FakeGithub([repo("app", spdx=None)], files, dirs) + self.assertEqual(audit.audit(gh, standard(), NOW), []) + + def test_a_repository_without_workflows_is_flagged(self): + files, _ = good_files("app") + files.pop((f"{ORG}/app", ".github/workflows/ci.yml")) + gh = FakeGithub([repo("app")], files, {}) + self.assertIn("has-ci", rules(audit.audit(gh, standard(), NOW))) + + def test_the_shared_gate_pin_must_be_a_full_sha_with_a_version_comment(self): + bad_pins = { + "branch name": "@5b96639ee3e06da9f4ab7603c59fd818643ade6c # main\n", + "no comment": "@150de5008682330747ba6880f1649b388858c206\n", + "short sha": "@150de50 # v0.1.0\n", + "moving tag": "@v0 # v0\n", + } + for label, pin in bad_pins.items(): + with self.subTest(label): + files, dirs = good_files("app") + files[(f"{ORG}/app", ".github/workflows/ci.yml")] = ( + " uses: naindev-contrast3d/ci-shared/.github/workflows/gate.yml" + + pin + ) + findings = audit.audit( + FakeGithub([repo("app")], files, dirs), standard(), NOW + ) + self.assertEqual( + (rules(findings), findings[0].severity), ({"gate-pin"}, "error") + ) + + def test_a_commented_reference_is_documentation_not_a_call(self): + files, dirs = good_files("app") + files[(f"{ORG}/app", ".github/workflows/ci.yml")] = ( + "# example:\n" + "# uses: naindev-contrast3d/ci-shared/.github/workflows/gate.yml@\n" + "jobs:\n test:\n runs-on: x\n" + ) + findings = audit.audit(FakeGithub([repo("app")], files, dirs), standard(), NOW) + self.assertEqual(rules(findings), {"shared-gate"}) + + def test_a_ci_that_does_not_call_the_shared_gate_is_a_warning(self): + files, dirs = good_files("app") + files[(f"{ORG}/app", ".github/workflows/ci.yml")] = ( + "jobs:\n test:\n runs-on: x\n" + ) + findings = audit.audit(FakeGithub([repo("app")], files, dirs), standard(), NOW) + self.assertEqual( + (rules(findings), findings[0].severity), ({"shared-gate"}, "warning") + ) + + def test_too_many_topic_branches_is_a_warning_with_the_count(self): + files, dirs = good_files("app") + names = ["main"] + [f"feat/{i}" for i in range(12)] + gh = FakeGithub([repo("app")], files, dirs, branches={f"{ORG}/app": names}) + findings = audit.audit(gh, standard(), NOW) + self.assertEqual(rules(findings), {"topic-branches"}) + self.assertIn("12", findings[0].message) + + def test_archived_repositories_are_skipped(self): + gh = FakeGithub([repo("old", delete_branch=False, archived=True)]) + self.assertEqual(audit.audit(gh, standard(), NOW), []) + + def test_the_organisation_must_require_two_factor_authentication(self): + findings = audit.audit(clean_world(two_fa=False), standard(), NOW) + org_findings = [f for f in findings if f.repo == ORG] + self.assertEqual( + ([f.rule for f in org_findings], org_findings[0].severity), + (["org-2fa"], "error"), + ) + + def test_members_with_an_insecure_second_factor_are_a_warning(self): + findings = audit.audit(clean_world(insecure=["david"]), standard(), NOW) + org_findings = [f for f in findings if f.repo == ORG] + self.assertEqual( + ([f.rule for f in org_findings], org_findings[0].severity), + (["org-2fa-method"], "warning"), + ) + self.assertIn("david", org_findings[0].message) + + +class Exceptions(unittest.TestCase): + def finding_world(self): + files, dirs = good_files("app") + return FakeGithub([repo("app", delete_branch=False)], files, dirs) + + def exception(self, expires): + return { + "repo": f"{ORG}/app", + "rule": "delete-branch-on-merge", + "reason": "kept on purpose", + "expires": expires, + } + + def test_a_valid_exception_silences_the_rule_for_that_repository(self): + std = standard(exceptions=[self.exception("2027-01-01")]) + self.assertEqual(audit.audit(self.finding_world(), std, NOW), []) + + def test_an_expired_exception_stops_applying_and_is_itself_a_finding(self): + std = standard(exceptions=[self.exception("2026-10-01")]) + found = {f.rule: f for f in audit.audit(self.finding_world(), std, NOW)} + self.assertEqual(set(found), {"delete-branch-on-merge", "exception-expired"}) + self.assertEqual(found["exception-expired"].severity, "error") + + def test_an_exception_for_another_repository_or_rule_changes_nothing(self): + other = dict(self.exception("2027-01-01"), repo=f"{ORG}/elsewhere") + std = standard( + exceptions=[other, dict(self.exception("2027-01-01"), rule="codeowners")] + ) + self.assertEqual( + rules(audit.audit(self.finding_world(), std, NOW)), + {"delete-branch-on-merge"}, + ) + + def test_an_exception_without_reason_or_date_is_rejected(self): + for bad in ( + {"repo": "a/b", "rule": "x"}, + {"repo": "a/b", "rule": "x", "reason": "r"}, + ): + with self.subTest(bad=bad), self.assertRaises(audit.StandardError): + audit.validate_standard(standard(exceptions=[bad])) + + +class Severity(unittest.TestCase): + def test_the_standard_can_change_the_severity_of_a_rule(self): + files, dirs = good_files("app") + gh = FakeGithub([repo("app", delete_branch=False)], files, dirs) + findings = audit.audit( + gh, standard(severity={"delete-branch-on-merge": "warning"}), NOW + ) + self.assertEqual(findings[0].severity, "warning") + + def test_an_unknown_severity_is_rejected(self): + with self.assertRaises(audit.StandardError): + audit.validate_standard(standard(severity={"codeowners": "fatal"})) + + def test_the_exit_code_follows_fail_on(self): + warning = [audit.Finding("o/a", "codeowners", "warning", "m")] + error = [audit.Finding("o/a", "license", "error", "m")] + self.assertEqual(audit.exit_code(warning, "error"), 0) + self.assertEqual(audit.exit_code(error, "error"), 1) + self.assertEqual(audit.exit_code(warning, "warning"), 1) + self.assertEqual(audit.exit_code([], "warning"), 0) + self.assertEqual(audit.exit_code(error, None), 0) + + +class Rendering(unittest.TestCase): + def test_findings_are_grouped_by_repository_and_ordered_by_severity(self): + findings = [ + audit.Finding("o/b", "codeowners", "warning", "no CODEOWNERS"), + audit.Finding("o/a", "license", "error", "license not recognised"), + audit.Finding("o/a", "codeowners", "warning", "no CODEOWNERS"), + ] + text = audit.render(findings, NOW) + self.assertLess(text.index("### o/a"), text.index("### o/b")) + self.assertLess( + text.index("license not recognised"), text.index("no CODEOWNERS") + ) + self.assertIn("- error `license`: license not recognised", text) + + def test_no_findings_says_so(self): + self.assertIn("No deviations", audit.render([], NOW)) + + def test_the_summary_counts_by_severity(self): + findings = [ + audit.Finding("o/a", "license", "error", "m"), + audit.Finding("o/a", "codeowners", "warning", "m"), + audit.Finding("o/b", "codeowners", "warning", "m"), + ] + self.assertIn("1 error, 2 warnings", audit.render(findings, NOW)) + + +class RealCommands(unittest.TestCase): + def setUp(self): + self.commands = [] + self.outputs = {} + self.errors = {} + self.gh = audit.Github(self.fake_run) + + def fake_run(self, command): + self.commands.append(command) + key = " ".join(command) + for needle, error in self.errors.items(): + if needle in key: + raise subprocess.CalledProcessError(1, command, stderr=error) + for needle, output in self.outputs.items(): + if needle in key: + return output + return "[]" + + def test_a_missing_file_reads_as_none(self): + self.errors["contents/.github/CODEOWNERS"] = "HTTP 404" + self.assertIsNone(self.gh.file_text(f"{ORG}/app", ".github/CODEOWNERS")) + + def test_a_file_is_read_through_the_raw_contents_media_type(self): + self.outputs["contents/.github/dependabot.yml"] = "version: 2\n" + self.assertEqual( + self.gh.file_text(f"{ORG}/app", ".github/dependabot.yml"), "version: 2\n" + ) + self.assertIn("application/vnd.github.raw+json", " ".join(self.commands[0])) + + def test_a_missing_directory_is_empty(self): + self.errors["contents/.github/workflows"] = "HTTP 404" + self.assertEqual(self.gh.dir_names(f"{ORG}/app", ".github/workflows"), []) + + def test_the_repositories_are_listed_with_the_fields_the_rules_read(self): + self.outputs["orgs/" + ORG + "/repos"] = json.dumps([repo("app")]) + self.assertEqual(self.gh.org_repos(ORG)[0]["name"], "app") + + def test_insecure_two_factor_members_are_listed_by_login(self): + self.outputs["2fa_insecure"] = json.dumps([{"login": "david"}]) + self.assertEqual(self.gh.insecure_two_factor_members(ORG), ["david"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ci_report.py b/tests/test_ci_report.py index 84a5a8f..06414c3 100644 --- a/tests/test_ci_report.py +++ b/tests/test_ci_report.py @@ -178,6 +178,25 @@ def test_a_repository_without_runs_has_no_rates_instead_of_dividing_by_zero(self self.assertIsNone(stats.median_queue_s) +class AuditSection(unittest.TestCase): + """`--audit` appends the standards audit to the report.""" + + def test_the_audit_module_is_found_beside_the_report(self): + module = report.load_audit() + self.assertTrue(callable(module.run)) + self.assertTrue(callable(module.render)) + + def test_the_section_is_the_audit_report_without_its_own_title(self): + module = report.load_audit() + findings = [module.Finding("o/a", "license", "error", "license not recognised")] + text = report.audit_section(module.render(findings, NOW)) + self.assertTrue(text.startswith("## Standards audit")) + self.assertIn("- error `license`: license not recognised", text) + self.assertNotIn( + "# Standards audit\n\n2026", text.replace("## Standards audit", "") + ) + + class Until(unittest.TestCase): """A window that ends in the past makes a baseline reproducible after later changes."""