diff --git a/README.md b/README.md index d04e4f4..1eeba40 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ Status: in progress. The design is in `specs/next-ci-reliability` of logs once any workflow run that has been queued longer than `queue_alert_minutes` (default 30): a runner that is offline or saturated otherwise holds jobs for up to 24 hours before they fail. The runs already reported are remembered in `queue-alerts.json` beside the configuration. + After every merge it comments `` on the pull request, so a merge made + around the train can be counted, and it writes a `heartbeat:` line (repositories, labelled pull + requests, version of the script) at most once an hour as proof of life. `--dry-run` changes nothing. Tests: `python3 -m unittest discover -s tests`. @@ -45,7 +48,11 @@ Status: in progress. The design is in `specs/next-ci-reliability` of never reached its steps, or the shared preflight) and not the code, the median duration and the median queue time, followed by a snapshot of today: runners online, offline and busy, queued runs and open pull requests. Use it to plan from the current state, not from memory: - `py -3 report/ci_report.py --config train/repos.json --repo Nain9Dev/crm-platform --days 7`. + `py -3 report/ci_report.py --config train/repos.json --repo naindev-contrast3d/contrast-hub --days 7`. + For the repositories of `--config` (the shared train) the snapshot also counts the pull requests + merged outside the train since `--train-since`, and `--train-dir D:\Development\Tools\ci-train` + adds a section on the installed train: age of its log (silent after 90 minutes) and whether its + script differs from the one in `ci-shared` main. `--until 2026-10-14T11:00` ends the period in the past, so a baseline can be reproduced after later changes; `--limit` raises the number of runs read per repository (default 300). diff --git a/report/ci_report.py b/report/ci_report.py index 9a3d112..6873cfd 100644 --- a/report/ci_report.py +++ b/report/ci_report.py @@ -17,6 +17,7 @@ import argparse import datetime as dt +import hashlib import importlib.util import json import statistics @@ -28,6 +29,14 @@ RUN_FIELDS = "databaseId,conclusion,status,workflowName,createdAt,startedAt,updatedAt,event,headBranch" INFRA_STEPS = ("Set up job", "Complete job") +# The comment the merge train writes on every pull request it merges (train/automerge.py). +TRAIN_MARK = "" +TRAIN_SILENT_AFTER_MIN = 90 # the train writes a heartbeat every 60 minutes + + +def text_version(text: str) -> str: + """A short fingerprint of a script, the same for LF and CRLF copies (as in the train).""" + return hashlib.sha256(text.replace("\r\n", "\n").encode("utf-8")).hexdigest()[:12] def run_gh(command: list[str]) -> str: @@ -91,6 +100,38 @@ def queued_runs(self, repo: str) -> list[dict]: ) return json.loads(out or "[]") + def merged_prs(self, repo: str, limit: int) -> list[dict]: + out = self._run( + [ + "gh", + "pr", + "list", + "--repo", + repo, + "--state", + "merged", + "--limit", + str(limit), + "--json", + "number,mergedAt,comments", + ] + ) + return json.loads(out or "[]") + + def file_text(self, repo: str, path: str) -> str | None: + try: + return self._run( + [ + "gh", + "api", + "-H", + "Accept: application/vnd.github.raw+json", + f"repos/{repo}/contents/{path}", + ] + ) + except subprocess.CalledProcessError: + return None + def open_prs(self, repo: str) -> list[dict]: out = self._run( [ @@ -126,6 +167,9 @@ class RepoStats: oldest_queued_min: int | None = None open_prs: int = 0 automerge_prs: int = 0 + # Only for repositories served by the shared merge train; None elsewhere. + merged_total: int | None = None + merged_by_hand: int | None = None @property def fail_rate(self) -> float | None: @@ -172,6 +216,8 @@ def collect( sample: int = 30, until: dt.datetime | None = None, limit: int = 300, + train_repos: frozenset[str] | set[str] = frozenset(), + train_since: dt.datetime | None = None, ) -> RepoStats: """Metrics for runs created in the `days` before `until` (default `now`). @@ -237,9 +283,89 @@ def collect( for p in prs if "automerge" in {label["name"] for label in p.get("labels", [])} ) + + if repo in train_repos: + floor = max(since, train_since) if train_since else since + merged = [ + m for m in gh.merged_prs(repo, 200) if floor <= _parse(m["mergedAt"]) <= end + ] + stats.merged_total = len(merged) + stats.merged_by_hand = sum( + 1 + for m in merged + if not any( + TRAIN_MARK in (c.get("body") or "") for c in m.get("comments", []) + ) + ) return stats +@dataclass +class TrainHealth: + log_age_min: ( + int | None + ) # minutes since the log was last written; None without a log + silent: bool # no log line for too long + copy: str # current, stale, missing or unknown + installed_version: str | None = None + repository_version: str | None = None + + +def train_health( + train_dir: Path, + gh, + now: dt.datetime, + repo: str = "naindev-contrast3d/ci-shared", +) -> TrainHealth: + """Is the installed train alive and is it the script of `ci-shared` main?""" + train_dir = Path(train_dir) + script = train_dir / "automerge.py" + log = train_dir / "train.log" + + age = None + if log.exists(): + modified = dt.datetime.fromtimestamp(log.stat().st_mtime, dt.UTC) + age = int((now - modified).total_seconds() // 60) + silent = age is not None and age > TRAIN_SILENT_AFTER_MIN + + if not script.exists(): + return TrainHealth(age, silent, "missing") + installed = text_version(script.read_text(encoding="utf-8")) + published = gh.file_text(repo, "train/automerge.py") + if published is None: + return TrainHealth(age, silent, "unknown", installed_version=installed) + current = text_version(published) + return TrainHealth( + age, silent, "current" if installed == current else "stale", installed, current + ) + + +def render_train(health: TrainHealth) -> str: + lines = ["## Merge train", ""] + if health.copy == "missing": + lines.append("- Installed copy not found (the train is not installed here).") + return "\n".join(lines) + if health.log_age_min is None: + lines.append("- No log yet.") + elif health.silent: + lines.append( + f"- Warning: no log line for {health.log_age_min} min " + f"(the train writes a heartbeat every 60 minutes)." + ) + else: + lines.append(f"- Last log activity {health.log_age_min} min ago.") + if health.copy == "current": + lines.append("- The installed script matches ci-shared main.") + elif health.copy == "stale": + lines.append( + f"- Warning: the installed script differs from ci-shared main " + f"(installed {health.installed_version}, repository {health.repository_version})." + ) + else: + lines.append("- Could not read the script of ci-shared main to compare.") + return "\n".join(lines) + + def fmt_duration(seconds: float | None) -> str: if seconds is None: return "-" @@ -302,6 +428,10 @@ def render( lines.append( f"- Open pull requests: {s.open_prs} ({s.automerge_prs} labelled automerge)" ) + if s.merged_by_hand is not None: + lines.append( + f"- Merged outside the train: {s.merged_by_hand} of {s.merged_total}" + ) lines.append("") return "\n".join(lines) @@ -348,13 +478,24 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="append the standards audit of the organisation", ) + parser.add_argument( + "--train-dir", + help="folder of the installed merge train (automerge.py and train.log): adds its health", + ) + parser.add_argument( + "--train-since", + help="UTC date the shared train started serving the repositories of --config; " + "earlier merges are not counted as made around it", + ) parser.add_argument("--output", help="write the report here instead of printing it") args = parser.parse_args(argv) repos = list(args.repo) + train_repos: set[str] = set() if args.config: data = json.loads(Path(args.config).read_text(encoding="utf-8")) - repos += [entry["repo"] for entry in data.get("repos", [])] + train_repos = {entry["repo"] for entry in data.get("repos", [])} + repos += sorted(train_repos) if not repos: print("give at least one --repo or a --config", file=sys.stderr) return 2 @@ -362,11 +503,24 @@ def main(argv: list[str] | None = None) -> int: now = dt.datetime.now(dt.UTC) gh = Github() until = parse_moment(args.until) if args.until else None + train_since = parse_moment(args.train_since) if args.train_since else None stats = [ - collect(gh, repo, args.workflow, args.days, now, until=until, limit=args.limit) + collect( + gh, + repo, + args.workflow, + args.days, + now, + until=until, + limit=args.limit, + train_repos=train_repos, + train_since=train_since, + ) for repo in repos ] text = render(stats, args.days, now, until=until) + if args.train_dir: + text += "\n\n" + render_train(train_health(Path(args.train_dir), gh, now)) if args.audit: audit = load_audit() standard_path = Path(audit.__file__).with_name("standards.json") diff --git a/tests/test_automerge.py b/tests/test_automerge.py index 4e7f5ac..e6d22fb 100644 --- a/tests/test_automerge.py +++ b/tests/test_automerge.py @@ -50,7 +50,10 @@ def __init__( merge_error=False, queue=None, queue_error=False, + comment_error=False, ): + self.comment_error = comment_error + self.comments = [] self.prs = prs or [] self.queue = queue or {} self.queue_error = queue_error @@ -90,6 +93,11 @@ def merge(self, repo, number, sha, method): if self.merge_error: raise subprocess.CalledProcessError(1, ["gh"], stderr="not allowed") + def comment(self, repo, number, body): + if self.comment_error: + raise subprocess.CalledProcessError(1, ["gh"], stderr="comment boom") + self.comments.append((repo, number, body)) + def update_branch(self, repo, number, sha): self.calls.append(("update", repo, number, sha)) return self.update_result @@ -101,12 +109,19 @@ def mutations(self): return [c for c in self.calls if c[0] != "open_prs"] -def run(gh, repos=None, dry_run=False, alerts=None, now=None): +def run(gh, repos=None, dry_run=False, alerts=None, now=None, heartbeat=None): logs = [] settings = train.Settings() configs = repos or [train.RepoConfig(repo=REPO)] code = train.run_cycle( - gh, configs, settings, dry_run=dry_run, log=logs.append, alerts=alerts, now=now + gh, + configs, + settings, + dry_run=dry_run, + log=logs.append, + alerts=alerts, + now=now, + heartbeat=heartbeat, ) return code, logs @@ -114,6 +129,75 @@ def run(gh, repos=None, dry_run=False, alerts=None, now=None): NOW = dt.datetime(2026, 10, 14, 13, 0, 0, tzinfo=dt.UTC) +class MergeMark(unittest.TestCase): + """The train marks what it merges, so a merge made around it can be counted (FR13).""" + + def test_a_merged_pull_request_gets_the_train_mark_with_its_head(self): + gh = FakeGithub(prs=[pr(7)]) + run(gh) + self.assertEqual(len(gh.comments), 1) + repo, number, body = gh.comments[0] + self.assertEqual((repo, number), (REPO, 7)) + self.assertIn(train.MERGE_MARK, body) + self.assertIn(SHA[:12], body) + + def test_nothing_is_marked_in_a_dry_run_or_when_the_merge_is_refused(self): + dry = FakeGithub(prs=[pr(7)]) + run(dry, dry_run=True) + refused = FakeGithub(prs=[pr(7)], merge_error=True) + run(refused) + self.assertEqual((dry.comments, refused.comments), ([], [])) + + def test_a_failing_comment_is_logged_and_does_not_undo_or_fail_the_merge(self): + gh = FakeGithub(prs=[pr(7)], comment_error=True) + code, logs = run(gh) + self.assertEqual(code, 0) + self.assertEqual(gh.mutations(), [("merge", REPO, 7, SHA, "merge")]) + self.assertTrue(any("mark failed" in line for line in logs)) + + +class HeartbeatLine(unittest.TestCase): + """A line at least once an hour proves the train is alive and says which script it runs (FR14).""" + + def state(self): + return Path(tempfile.mkdtemp()) / "heartbeat.json" + + def test_the_first_cycle_logs_a_heartbeat_with_counts_and_the_script_version(self): + gh = FakeGithub(prs=[pr(1), pr(2, "b" * 40)], behind=2) + _, logs = run(gh, heartbeat=train.Heartbeat(self.state()), now=NOW) + line = next(item for item in logs if "heartbeat" in item) + self.assertIn("1 repositories", line) + self.assertIn("2 labelled pull requests", line) + self.assertIn(train.script_version(), line) + + def test_no_second_heartbeat_within_the_hour_but_one_after_it(self): + path = self.state() + gh = FakeGithub(prs=[]) + _, first = run(gh, heartbeat=train.Heartbeat(path), now=NOW) + _, soon = run( + gh, heartbeat=train.Heartbeat(path), now=NOW + dt.timedelta(minutes=59) + ) + _, later = run( + gh, heartbeat=train.Heartbeat(path), now=NOW + dt.timedelta(minutes=61) + ) + counts = [ + sum("heartbeat" in item for item in logs) for logs in (first, soon, later) + ] + self.assertEqual(counts, [1, 0, 1]) + + def test_a_dry_run_does_not_write_the_heartbeat_state(self): + path = self.state() + run(FakeGithub(prs=[]), dry_run=True, heartbeat=train.Heartbeat(path), now=NOW) + self.assertFalse(path.exists()) + + def test_the_script_version_is_the_same_for_lf_and_crlf_copies(self): + text = "a\nb\n" + self.assertEqual( + train.text_version(text), train.text_version(text.replace("\n", "\r\n")) + ) + self.assertNotEqual(train.text_version(text), train.text_version(text + "c\n")) + + def queued(run_id, minutes_ago, branch="feat/x", workflow="CI"): created = NOW - dt.timedelta(minutes=minutes_ago) return { @@ -440,6 +524,13 @@ def test_merge_pins_the_head_commit_and_makes_a_merge_commit(self): ], ) + def test_comment_posts_the_body_on_the_pull_request(self): + self.gh.comment(REPO, 9, f"{train.MERGE_MARK}\nMerged") + self.assertEqual( + self.commands[0][:7], ["gh", "pr", "comment", "9", "--repo", REPO, "--body"] + ) + self.assertIn(train.MERGE_MARK, self.commands[0][7]) + def test_queued_runs_asks_for_queued_status_and_parses_json(self): self.outputs["run list"] = json.dumps([queued(5, 40)]) runs = self.gh.queued_runs(REPO) diff --git a/tests/test_ci_report.py b/tests/test_ci_report.py index 06414c3..38c950a 100644 --- a/tests/test_ci_report.py +++ b/tests/test_ci_report.py @@ -77,6 +77,12 @@ def queued_runs(self, repo): def open_prs(self, repo): return self._prs + def merged_prs(self, repo, limit): + return getattr(self, "merged", []) + + def file_text(self, repo, path): + return getattr(self, "files", {}).get((repo, path)) + def sample_github(): runs = [ @@ -178,6 +184,145 @@ def test_a_repository_without_runs_has_no_rates_instead_of_dividing_by_zero(self self.assertIsNone(stats.median_queue_s) +MARK = "" + + +def merged_pr(number, minutes_ago, marked): + comments = ( + [{"body": f"{MARK}\nMerged by the merge train"}] + if marked + else [{"body": "lgtm"}] + ) + return {"number": number, "mergedAt": iso(minutes_ago), "comments": comments} + + +class MergedOutsideTheTrain(unittest.TestCase): + """A merged pull request without the train's mark was merged around the train (FR13).""" + + def stats(self, merged, train_repos=(REPO,), train_since=None): + gh = sample_github() + gh.merged = merged + return report.collect( + gh, + REPO, + "CI", + days=7, + now=NOW, + train_repos=set(train_repos), + train_since=train_since, + ) + + def test_unmarked_merges_inside_the_window_are_counted(self): + stats = self.stats( + [merged_pr(1, 60, True), merged_pr(2, 120, False), merged_pr(3, 180, False)] + ) + self.assertEqual((stats.merged_total, stats.merged_by_hand), (3, 2)) + + def test_merges_older_than_the_window_or_before_the_train_are_not_counted(self): + old = merged_pr( + 4, 60 * 24 * 10, False + ) # ten days ago, outside the 7-day window + before_train = merged_pr(5, 60 * 24 * 3, False) + since = NOW - dt.timedelta(days=1) + stats = self.stats( + [old, before_train, merged_pr(6, 30, True)], train_since=since + ) + self.assertEqual((stats.merged_total, stats.merged_by_hand), (1, 0)) + + def test_a_repository_without_the_shared_train_is_not_counted_at_all(self): + stats = self.stats([merged_pr(1, 60, False)], train_repos=()) + self.assertIsNone(stats.merged_by_hand) + self.assertIsNone(stats.merged_total) + + def test_the_snapshot_says_how_many_were_merged_around_the_train(self): + stats = self.stats([merged_pr(1, 60, True), merged_pr(2, 120, False)]) + text = report.render([stats], days=7, now=NOW) + self.assertIn("Merged outside the train: 1 of 2", text) + + def test_the_real_query_asks_for_merged_pull_requests_with_their_comments(self): + commands = [] + gh = report.Github(lambda command: commands.append(command) or "[]") + gh.merged_prs(REPO, 100) + self.assertIn("merged", commands[0]) + self.assertTrue(any("comments" in part for part in commands[0])) + + def test_the_mark_is_the_same_text_the_train_writes(self): + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "train")) + import automerge as train + + self.assertEqual(report.TRAIN_MARK, train.MERGE_MARK) + + +class TrainHealth(unittest.TestCase): + """The report shows whether the train is alive and runs the current script (FR14).""" + + def setUp(self): + import os + import tempfile + + self.os = os + self.dir = Path(tempfile.mkdtemp()) + self.gh = FakeGithub([], {}) + self.gh.files = { + ("naindev-contrast3d/ci-shared", "train/automerge.py"): "script v1\n" + } + + def write(self, name, text, minutes_ago=None): + path = self.dir / name + path.write_text(text, encoding="utf-8", newline="") + if minutes_ago is not None: + stamp = (NOW - dt.timedelta(minutes=minutes_ago)).timestamp() + self.os.utime(path, (stamp, stamp)) + + def health(self): + return report.train_health(self.dir, self.gh, NOW) + + def test_a_recent_log_and_the_current_script_are_healthy(self): + self.write("train.log", "[x] heartbeat\n", minutes_ago=12) + self.write("automerge.py", "script v1\n") + health = self.health() + self.assertEqual(health.log_age_min, 12) + self.assertFalse(health.silent) + self.assertEqual(health.copy, "current") + + def test_a_log_older_than_ninety_minutes_means_the_train_is_silent(self): + self.write("train.log", "[x] heartbeat\n", minutes_ago=130) + self.write("automerge.py", "script v1\n") + self.assertTrue(self.health().silent) + + def test_a_copy_that_differs_from_ci_shared_main_is_stale(self): + self.write("train.log", "x\n", minutes_ago=5) + self.write("automerge.py", "script v0\n") + health = self.health() + self.assertEqual(health.copy, "stale") + self.assertNotEqual(health.installed_version, health.repository_version) + + def test_line_endings_alone_do_not_make_a_copy_stale(self): + self.write("train.log", "x\n", minutes_ago=5) + self.write("automerge.py", "script v1\r\n") + self.assertEqual(self.health().copy, "current") + + def test_a_missing_folder_is_reported_as_not_installed(self): + health = report.train_health(self.dir / "nowhere", self.gh, NOW) + self.assertEqual(health.copy, "missing") + self.assertIsNone(health.log_age_min) + + def test_the_section_states_each_finding_in_words(self): + self.write("train.log", "x\n", minutes_ago=130) + self.write("automerge.py", "script v0\n") + text = report.render_train(self.health()) + self.assertIn("## Merge train", text) + self.assertIn("no log line for 130 min", text) + self.assertIn("differs from ci-shared main", text) + + def test_the_version_function_matches_the_trains(self): + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "train")) + import automerge as train + + for sample in ("a\nb\n", "a\r\nb\r\n", ""): + self.assertEqual(report.text_version(sample), train.text_version(sample)) + + class AuditSection(unittest.TestCase): """`--audit` appends the standards audit to the report.""" diff --git a/train/automerge.py b/train/automerge.py index 80d1856..186bd33 100644 --- a/train/automerge.py +++ b/train/automerge.py @@ -27,6 +27,7 @@ import argparse import datetime as dt +import hashlib import json import re import subprocess @@ -221,6 +222,9 @@ def merge(self, repo: str, number: int, sha: str, method: str = "merge") -> None ] ) + def comment(self, repo: str, number: int, body: str) -> None: + self._run(["gh", "pr", "comment", str(number), "--repo", repo, "--body", body]) + def update_branch(self, repo: str, number: int, sha: str) -> str: try: self._run( @@ -301,6 +305,50 @@ def classify(pr: dict, cfg: RepoConfig, settings: Settings, gh) -> tuple[str, st return "ready", "ready" +# Written as a comment on every pull request the train merges; the health report counts the merged +# pull requests that lack it as merged outside the train. +MERGE_MARK = "" + + +def text_version(text: str) -> str: + """A short fingerprint of a script, the same for LF and CRLF copies.""" + normalised = text.replace("\r\n", "\n") + return hashlib.sha256(normalised.encode("utf-8")).hexdigest()[:12] + + +def script_version() -> str: + return text_version(Path(__file__).read_text(encoding="utf-8")) + + +class Heartbeat: + """Decides when the train writes its hourly proof of life, and remembers the last one.""" + + EVERY = dt.timedelta(minutes=60) + + def __init__(self, path: Path | str | None): + self.path = Path(path) if path else None + self._last: dt.datetime | None = None + if self.path is not None and self.path.exists(): + try: + self._last = dt.datetime.fromisoformat( + json.loads(self.path.read_text(encoding="utf-8"))["last"] + ) + except (OSError, ValueError, KeyError, TypeError): + self._last = ( + None # an unreadable state only means one heartbeat too many + ) + + def due(self, now: dt.datetime) -> bool: + return self._last is None or now - self._last >= self.EVERY + + def mark(self, now: dt.datetime) -> None: + self._last = now + if self.path is not None: + self.path.write_text( + json.dumps({"last": now.isoformat()}), encoding="utf-8" + ) + + class QueueAlerts: """Remembers which queued runs were already reported, so each one is logged once.""" @@ -355,10 +403,12 @@ def run_cycle( log: Callable[[str], None], alerts: QueueAlerts | None = None, now: dt.datetime | None = None, + heartbeat: Heartbeat | None = None, ) -> int: """One pass over every repository. Returns 0 when nothing went wrong, 1 otherwise.""" code = 0 moment = now or dt.datetime.now(dt.UTC) + seen: list[int] = [] for cfg in configs: repo_log = lambda line, r=cfg.repo: log(f"{r} {line}") if alerts is not None: @@ -382,16 +432,32 @@ def run_cycle( detail = (getattr(exc, "stderr", "") or str(exc)).strip()[:200] repo_log(f"queue check failed: {detail}") try: - _repo_cycle(gh, cfg, settings, dry_run, repo_log) + _repo_cycle(gh, cfg, settings, dry_run, repo_log, seen) except (subprocess.CalledProcessError, OSError, ValueError, KeyError) as exc: detail = (getattr(exc, "stderr", "") or str(exc)).strip()[:300] log(f"{cfg.repo}: failed: {detail}") code = 1 + if heartbeat is not None and heartbeat.due(moment): + log( + f"heartbeat: {len(configs)} repositories, {sum(seen)} labelled pull requests, " + f"script {script_version()}" + ) + if not dry_run: + heartbeat.mark(moment) return code -def _repo_cycle(gh, cfg: RepoConfig, settings: Settings, dry_run: bool, log) -> None: +def _repo_cycle( + gh, + cfg: RepoConfig, + settings: Settings, + dry_run: bool, + log, + seen: list[int] | None = None, +) -> None: prs = gh.open_prs(cfg.repo, settings.label) + if seen is not None: + seen.append(len(prs)) # for the heartbeat line if not prs: return states = {} @@ -429,6 +495,16 @@ def _repo_cycle(gh, cfg: RepoConfig, settings: Settings, dry_run: bool, log) -> log(f"#{number}: merge refused: {(exc.stderr or '').strip()[:300]}") raise log(f"#{number}: merged {pr['headRefOid'][:12]}") + # The mark tells apart a merge made by the train from one made around it (FR13). The merge + # is already done: a failing comment is logged, never an error of the cycle. + try: + gh.comment( + cfg.repo, + number, + f"{MERGE_MARK}\nMerged by the merge train at head `{pr['headRefOid'][:12]}`.", + ) + except subprocess.CalledProcessError as exc: + log(f"#{number}: mark failed: {(exc.stderr or '').strip()[:200]}") elif stale: number, pr = stale[0] if dry_run: @@ -473,8 +549,15 @@ def log(message: str) -> None: log(f"configuration error: {exc}") return 2 alerts = QueueAlerts(Path(args.config).with_name("queue-alerts.json")) + heartbeat = Heartbeat(Path(args.config).with_name("heartbeat.json")) return run_cycle( - Github(), configs, settings, dry_run=args.dry_run, log=log, alerts=alerts + Github(), + configs, + settings, + dry_run=args.dry_run, + log=log, + alerts=alerts, + heartbeat=heartbeat, )