From 6fe9e5cff2ea3f767e96b7e88ffd90aac50c7e28 Mon Sep 17 00:00:00 2001 From: naindev <193035803+Nain9Dev@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:00:43 +0200 Subject: [PATCH] feat(report): write the weekly CI report to a file and toast only on a deviation The report and the runner watchdog existed, but the report was only produced when someone asked for it. weekly_report.py reads every private, non-archived repository of the organisation and writes ci-report-.md and latest.md. A week with a failure rate above 25 percent over at least five runs, a p90 wait for a runner above five minutes, or an infrastructure share of failures above a fifth puts the deviations at the top and raises one toast; a healthy week raises none. An unreadable repository is named in the report and skipped. install-weekly-report.ps1 registers it as a task for Mondays at 08:00. --- README.md | 9 ++ report/install-weekly-report.ps1 | 69 +++++++++++ report/weekly_report.py | 171 +++++++++++++++++++++++++++ tests/test_weekly_report.py | 191 +++++++++++++++++++++++++++++++ 4 files changed, 440 insertions(+) create mode 100644 report/install-weekly-report.ps1 create mode 100644 report/weekly_report.py create mode 100644 tests/test_weekly_report.py diff --git a/README.md b/README.md index ace11f4..3eeaab7 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,15 @@ Status: in progress. The design is in `specs/next-ci-reliability` of adds a section on the installed train: age of its log (silent after 90 minutes) and whether its script differs from the one in `ci-shared` main. +- `report/weekly_report.py` (available): the weekly report to a file. It runs `ci_report.py` over every + private repository of the organisation that is not archived and writes `ci-report-.md` (and + `latest.md`) to `--out-dir`. When the week deviates, with a failure rate above 25 % over at least + five runs, a p90 wait for a runner above five minutes, or more than a fifth of the failures caused by + the infrastructure, the deviations go at the top of the report and one toast is raised through + `--notify`; a healthy week raises nothing. The thresholds are options. `report/install-weekly-report.ps1` + registers it as the task `CI weekly report` (Mondays 08:00), from a normal PowerShell window of the + owner's session. + - `report/runner_watch.py` (available): the runner watchdog. The report says how the queue behaved over a period; this says that it is wrong now. Run from a scheduled task every few minutes (`py -3 report/runner_watch.py --org naindev-contrast3d --notify `), it reads diff --git a/report/install-weekly-report.ps1 b/report/install-weekly-report.ps1 new file mode 100644 index 0000000..363c043 --- /dev/null +++ b/report/install-weekly-report.ps1 @@ -0,0 +1,69 @@ +# Installs the weekly CI report on the desktop: every Monday at 08:00 it writes the report of the +# organisation to a file and raises a Windows toast only when the week deviates. Run it from a normal +# PowerShell window of the owner's session: scheduled tasks created from a sandboxed or elevated shell +# can be private to that shell. +# +# powershell -NoProfile -ExecutionPolicy Bypass -File report\install-weekly-report.ps1 +# powershell -NoProfile -ExecutionPolicy Bypass -File report\install-weekly-report.ps1 -Remove +# +# The scripts are copied to $Destination, so the task does not depend on which branch a checkout is on; +# run this again after weekly_report.py changes. The task uses the owner's `gh` login; nothing else is +# stored. Reports land in $Destination\reports as ci-report-.md, with a copy as latest.md. +param( + [string]$Organisation = 'naindev-contrast3d', + [string]$Destination = 'D:\Development\Tools\ci-train\weekly', + [string]$Notify = 'D:\Development\Tools\workstation-maintenance\notify.ps1', + [switch]$Remove +) + +$ErrorActionPreference = 'Stop' +$taskName = 'CI weekly report' + +if ($Remove) { + Unregister-ScheduledTask -TaskName $taskName -Confirm:$false -ErrorAction SilentlyContinue + Write-Host "Task '$taskName' removed. Files in $Destination are left in place." + exit 0 +} + +if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { throw 'gh is not installed' } +gh auth status *> $null +if ($LASTEXITCODE -ne 0) { throw 'gh is not signed in: run gh auth login' } +if (-not (Test-Path $Notify)) { throw "notify.ps1 not found at $Notify (pass -Notify)" } + +# The real pythonw.exe, not the `pyw` launcher (see train\install.ps1 for why). +$python = (py -3 -c "import sys;print(sys.executable)") +$pythonw = Join-Path (Split-Path $python) 'pythonw.exe' +if (-not (Test-Path $pythonw)) { throw "pythonw.exe not found at $pythonw" } + +New-Item -ItemType Directory -Force -Path $Destination | Out-Null +foreach ($file in 'weekly_report.py', 'runner_watch.py', 'ci_report.py') { + Copy-Item -Force (Join-Path $PSScriptRoot $file) (Join-Path $Destination $file) +} +$script = Join-Path $Destination 'weekly_report.py' +$reports = Join-Path $Destination 'reports' + +$arguments = "`"$script`" --org $Organisation --out-dir `"$reports`" --notify `"$Notify`"" +$action = New-ScheduledTaskAction -Execute $pythonw -Argument $arguments +$trigger = New-ScheduledTaskTrigger -Weekly -DaysOfWeek Monday -At 8:00 +$settings = New-ScheduledTaskSettingsSet ` + -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 15) ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable +$principal = New-ScheduledTaskPrincipal -UserId "$env:USERDOMAIN\$env:USERNAME" ` + -LogonType Interactive -RunLevel Limited + +Register-ScheduledTask ` + -TaskName $taskName ` + -Action $action ` + -Trigger $trigger ` + -Settings $settings ` + -Principal $principal ` + -Description "Writes the CI report of $Organisation to a file every Monday and toasts when the week deviates." ` + -Force | Out-Null + +Write-Host "Task '$taskName' installed: every Monday at 08:00 (or at the next start if the desktop was off)." +Write-Host " Reports: $reports" +Write-Host " Try now: py -3 `"$script`" --org $Organisation --out-dir `"$reports`"" +Write-Host " Remove: report\install-weekly-report.ps1 -Remove" diff --git a/report/weekly_report.py b/report/weekly_report.py new file mode 100644 index 0000000..021bce4 --- /dev/null +++ b/report/weekly_report.py @@ -0,0 +1,171 @@ +"""Weekly CI report to a file, with a toast only when the week deviates. + + py -3 weekly_report.py --org naindev-contrast3d --out-dir D:\\...\\reports --notify D:\\...\\notify.ps1 + +Run it from a scheduled task once a week. It reads, over `gh`, every private repository of the +organisation that is not archived and writes `ci-report-.md` (and a copy as `latest.md`) in the +format of `ci_report.py`: failure rate by cause, durations, queue time per job and a snapshot of today. +A week that deviates (a failure rate, a queue or an infrastructure share above its threshold) puts the +deviations at the top of the report and raises one toast; a healthy week raises nothing, because a +notification that always fires is one nobody reads. An unreadable repository is skipped and said in the +report, never fatal. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from typing import NamedTuple + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import ci_report +import runner_watch + +FAIL_RATE_MAX = 0.25 +WAIT_P90_MAX_S = 300 +INFRA_SHARE_MAX = 0.2 +MIN_RUNS = 5 # below this, a rate is noise +MIN_FAILURES = 3 # below this, an infrastructure share is noise + + +class Result(NamedTuple): + path: Path + deviations: list[str] + + +def find_deviations( + stats: list[ci_report.RepoStats], + fail_rate_max: float = FAIL_RATE_MAX, + wait_p90_max: float = WAIT_P90_MAX_S, + infra_share_max: float = INFRA_SHARE_MAX, + min_runs: int = MIN_RUNS, +) -> list[str]: + """One sentence per repository and finding that is over its threshold.""" + found: list[str] = [] + for s in stats: + rate = s.fail_rate + if rate is not None and s.conclusive >= min_runs and rate > fail_rate_max: + found.append( + f"{s.repo}: {ci_report._pct(rate)} of the runs failed ({s.failed} of {s.conclusive})" + ) + if s.job_wait_p90_s is not None and s.job_wait_p90_s > wait_p90_max: + found.append( + f"{s.repo}: the p90 wait for a runner was {ci_report.fmt_duration(s.job_wait_p90_s)}" + ) + share = s.infra_share + if share is not None and s.failed >= MIN_FAILURES and share > infra_share_max: + found.append( + f"{s.repo}: {s.infra_failures} of {s.failed} failures were the infrastructure" + ) + return found + + +def collect_all(gh, org: str, days: int, now: dt.datetime, limit: int): + """Stats of every private, non-archived repository; the unreadable ones are named apart.""" + stats: list[ci_report.RepoStats] = [] + skipped: list[str] = [] + for repo in gh.org_repos(org): + if repo.get("isArchived") or repo.get("visibility") != "PRIVATE": + continue + scope = f"{org}/{repo['name']}" + try: + stats.append(ci_report.collect(gh, scope, "CI", days, now, limit=limit)) + except (subprocess.CalledProcessError, ValueError, KeyError) as exc: + skipped.append(f"{scope}: {type(exc).__name__}") + return stats, skipped + + +def run( + gh, + org: str, + out_dir: Path, + now: dt.datetime, + *, + days: int = 7, + limit: int = 300, + notify_path: str | None = None, + fail_rate_max: float = FAIL_RATE_MAX, + wait_p90_max: float = WAIT_P90_MAX_S, + infra_share_max: float = INFRA_SHARE_MAX, + runner: Callable[[list[str]], object] = runner_watch._run_hidden, +) -> Result: + stats, skipped = collect_all(gh, org, days, now, limit) + deviations = find_deviations(stats, fail_rate_max, wait_p90_max, infra_share_max) + text = ci_report.render(stats, days, now) + head = "" + if deviations: + head = "## Deviations\n\n" + "\n".join(f"- {d}" for d in deviations) + "\n\n" + if skipped: + head += "## Not read\n\n" + "\n".join(f"- {s}" for s in skipped) + "\n\n" + if head: + first, _, rest = text.partition("\n\n") + text = first + "\n\n" + head + rest + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / f"ci-report-{now.strftime('%Y-%m-%d')}.md" + path.write_text(text, encoding="utf-8", newline="\n") + (out_dir / "latest.md").write_text(text, encoding="utf-8", newline="\n") + if deviations and notify_path: + shown = "; ".join(deviations[:3]) + (" ..." if len(deviations) > 3 else "") + runner_watch.notify( + [ + runner_watch.Alert( + f"weekly:{now.strftime('%Y-%m-%d')}", + f"CI weekly report: {len(deviations)} deviation(s)", + shown, + ) + ], + notify_path, + runner, + ) + return Result(path, deviations) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--org", required=True) + parser.add_argument( + "--out-dir", + help="where the reports go (default: a reports folder beside this file)", + ) + parser.add_argument("--days", type=int, default=7) + parser.add_argument( + "--limit", type=int, default=300, help="runs read per repository" + ) + parser.add_argument( + "--notify", help="path of notify.ps1: one toast when the week deviates" + ) + parser.add_argument("--fail-rate-max", type=float, default=FAIL_RATE_MAX) + parser.add_argument( + "--wait-p90-max", type=float, default=WAIT_P90_MAX_S, help="seconds" + ) + parser.add_argument("--infra-share-max", type=float, default=INFRA_SHARE_MAX) + args = parser.parse_args(argv) + out_dir = ( + Path(args.out_dir) + if args.out_dir + else Path(__file__).resolve().with_name("reports") + ) + result = run( + runner_watch.Github(), + args.org, + out_dir, + dt.datetime.now(dt.UTC), + days=args.days, + limit=args.limit, + notify_path=args.notify, + fail_rate_max=args.fail_rate_max, + wait_p90_max=args.wait_p90_max, + infra_share_max=args.infra_share_max, + ) + print(f"report: {result.path}") + for deviation in result.deviations: + print(f"deviation: {deviation}") + return 1 if result.deviations else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_weekly_report.py b/tests/test_weekly_report.py new file mode 100644 index 0000000..a241137 --- /dev/null +++ b/tests/test_weekly_report.py @@ -0,0 +1,191 @@ +"""Tests for report/weekly_report.py: the report written to a file, and a toast only for a deviation.""" + +from __future__ import annotations + +import datetime as dt +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "report")) + +import ci_report +import weekly_report as weekly + +NOW = dt.datetime(2026, 10, 19, 8, 0, 0, tzinfo=dt.UTC) # a Monday +ORG = "acme" + + +def stats(repo="acme/a", conclusive=20, failed=2, infra=0, p90=None): + s = ci_report.RepoStats( + repo=repo, conclusive=conclusive, failed=failed, infra_failures=infra + ) + s.job_wait_p90_s = p90 + s.jobs_sampled = 1 if p90 is not None else 0 + return s + + +class Deviations(unittest.TestCase): + def test_a_healthy_repository_raises_nothing(self): + self.assertEqual(weekly.find_deviations([stats(failed=2, p90=60)]), []) + + def test_a_high_failure_rate_is_a_deviation(self): + found = weekly.find_deviations([stats(conclusive=20, failed=6)]) + self.assertEqual(len(found), 1) + self.assertIn("acme/a", found[0]) + self.assertIn("30%", found[0]) + + def test_a_few_runs_prove_nothing(self): + # Two failures out of three is noise: below the minimum number of runs nothing is judged. + self.assertEqual(weekly.find_deviations([stats(conclusive=3, failed=2)]), []) + + def test_a_long_queue_is_a_deviation(self): + found = weekly.find_deviations([stats(p90=601)]) + self.assertEqual(len(found), 1) + self.assertIn("10m01s", found[0]) + + def test_infrastructure_failures_are_a_deviation_of_their_own(self): + found = weekly.find_deviations([stats(conclusive=20, failed=4, infra=3)]) + self.assertTrue(any("infrastructure" in f for f in found)) + + def test_the_thresholds_are_parameters(self): + self.assertEqual( + len(weekly.find_deviations([stats(failed=2)], fail_rate_max=0.05)), 1 + ) + self.assertEqual(weekly.find_deviations([stats(p90=601)], wait_p90_max=900), []) + + def test_a_repository_without_data_is_not_judged(self): + self.assertEqual(weekly.find_deviations([stats(conclusive=0, failed=0)]), []) + + +class FakeGithub: + """Just enough of the reads `ci_report.collect` makes.""" + + def __init__(self, failing=False): + self.failing = failing + + def org_repos(self, org): + return [ + {"name": "a", "visibility": "PRIVATE", "isArchived": False}, + {"name": "old", "visibility": "PRIVATE", "isArchived": True}, + {"name": "open", "visibility": "PUBLIC", "isArchived": False}, + ] + + def runs(self, repo, limit): + out = [] + for i in range(10): + conclusion = "failure" if (self.failing and i < 6) else "success" + moment = (NOW - dt.timedelta(hours=i + 1)).strftime("%Y-%m-%dT%H:%M:%SZ") + out.append( + { + "databaseId": i + 1, + "conclusion": conclusion, + "status": "completed", + "workflowName": "CI", + "createdAt": moment, + "startedAt": moment, + "updatedAt": moment, + "event": "push", + "headBranch": "main", + } + ) + return out + + def jobs(self, repo, run_id): + return [ + { + "name": "Verify", + "conclusion": "failure", + "startedAt": None, + "steps": [{"name": "Tests", "conclusion": "failure"}], + } + ] + + def job_timings(self, repo, run_id): + return [] + + def runners(self, repo): + return [] + + def queued_runs(self, repo): + return [] + + def open_prs(self, repo): + return [] + + +class Running(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.out = Path(self.tmp.name) + self.toasts = [] + + def tearDown(self): + self.tmp.cleanup() + + def go(self, gh, notify_path=r"C:\x\notify.ps1"): + return weekly.run( + gh, + ORG, + self.out, + NOW, + days=7, + notify_path=notify_path, + runner=self.toasts.append, + ) + + def test_the_report_is_written_with_the_date_in_its_name_and_as_latest(self): + result = self.go(FakeGithub()) + dated = self.out / "ci-report-2026-10-19.md" + self.assertTrue(dated.exists()) + self.assertEqual( + (self.out / "latest.md").read_text(encoding="utf-8"), + dated.read_text(encoding="utf-8"), + ) + text = dated.read_text(encoding="utf-8") + self.assertIn("# CI health report", text) + self.assertIn("acme/a", text) + self.assertEqual(result.path, dated) + + def test_only_private_repositories_that_are_not_archived_are_reported(self): + self.go(FakeGithub()) + text = (self.out / "latest.md").read_text(encoding="utf-8") + self.assertNotIn("acme/old", text) + self.assertNotIn("acme/open", text) + + def test_a_healthy_week_writes_the_report_and_raises_no_toast(self): + result = self.go(FakeGithub()) + self.assertEqual(result.deviations, []) + self.assertEqual(self.toasts, []) + + def test_a_bad_week_raises_one_toast_naming_the_repositories(self): + result = self.go(FakeGithub(failing=True)) + self.assertGreaterEqual(len(result.deviations), 1) + self.assertEqual(len(self.toasts), 1) + command = self.toasts[0] + self.assertIn("-File", command) + self.assertIn("acme/a", command[command.index("-Body") + 1]) + + def test_without_a_notify_path_nothing_is_toasted(self): + self.go(FakeGithub(failing=True), notify_path=None) + self.assertEqual(self.toasts, []) + + def test_a_failing_toast_never_stops_the_report(self): + def boom(command): + raise OSError("no powershell") + + result = weekly.run( + FakeGithub(failing=True), + ORG, + self.out, + NOW, + days=7, + notify_path="n.ps1", + runner=boom, + ) + self.assertTrue(result.path.exists()) + + +if __name__ == "__main__": + unittest.main()