fix(checks): derive preflight's gate matrix from every workflow that calls testing-conventions

.github/workflows/internals-checks-ci.yml

@@ -32,3 +32,22 @@ jobs:
       config: internals/checks/testing-conventions.toml
       base: ${{ github.event.pull_request.base.sha || 'origin/main' }}
       gates: '["colocated-test", "unit-lint", "integration-lint", "unit-coverage", "mutation", "e2e-verify"]'
+
+  internals-checks-integration:
+    # The conventions job runs this package's colocated unit tier and checks its
+    # e2e attestation; neither runs tests/integration/, the tier that reads the
+    # repo's real workflow files. Without it a workflow rename can delete the
+    # ground truth a check reads and no job notices (#973).
+    name: internals/checks integration tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.11"
+
+      - name: Run integration tests
+        run: uv run python -m pytest tests/integration/ -x -q --tb=short
+        working-directory: internals/checks

internals/checks/README.md

@@ -23,15 +23,17 @@ Each check follows the same shape:
   raising `SystemExit` with its return code.
 
 Every module (except empty `__init__.py`s) carries a colocated `*_test.py`, gated by
-`conventions.yml`'s `internals-checks` job: `colocated-test`, `unit-lint`, `unit-coverage`,
+`internals-checks-ci.yml`'s `internals-checks` job: `colocated-test`, `unit-lint`, `unit-coverage`,
 `mutation`, and `e2e-verify`.
 
 ## Test tiers
 
 - `tests/integration/` -- exercises each check's `gate.run()` against real collaborators (real
-  `git`, real pytest subprocess) rather than the packaged CLI. Gated by the `integration-lint` gate
-  in `conventions.yml`'s `internals-checks` job, which derives `tests/integration/` from the package
-  root (#417).
+  `git`, real pytest subprocess, the repo's own workflow files) rather than the packaged CLI. Gated
+  by the `integration-lint` gate in `internals-checks-ci.yml`'s `internals-checks` job, which
+  derives `tests/integration/` from the package root (#417), and *run* by that workflow's
+  `internals-checks-integration` job -- the tier reads real repo state, so a rename that a colocated
+  fixture cannot see fails here instead (#973).
 - `tests/e2e/` -- spawns the real `dirsql-checks` CLI as a subprocess with nothing mocked. Not run
   in CI; gated only via `e2e-attestation.json` freshness (see AGENTS.md, "E2E Attestation").
 

internals/checks/e2e-attestations/claude-tackle-973-4e1epu.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e -x -q",
+  "ran_at": 1787253882,
+  "exit_code": 0,
+  "commit": "4ffd060d6a90862e0a96b71b28240c31aacc3c5a",
+  "branch": "claude/tackle-973-4e1epu"
+}

internals/checks/src/checks/preflight/cli.py

@@ -1,7 +1,8 @@
 """The preflight check -- repo-only (#781).
 
 Backs `dirsql-checks preflight`: derive the testing-conventions gate matrix from
-`.github/workflows/conventions.yml` and run every pair locally.
+the workflows in `.github/workflows/` that call the reusable workflow, and run
+every pair locally.
 """
 
 from __future__ import annotations
@@ -11,13 +12,18 @@
 import click
 
 from .gate import default_runner, read_e2e, run
+from .matrix import NoGateMatrix, WORKFLOWS, sources
 
 
 @click.command()
 @click.option(
     "--conventions",
-    default=".github/workflows/conventions.yml",
-    help="Workflow whose reusable-workflow callers define the gate matrix.",
+    "conventions",
+    multiple=True,
+    help=(
+        "Workflow whose reusable-workflow callers define the gate matrix (repeatable). "
+        f"Default: every caller in {WORKFLOWS}."
+    ),
 )
 @click.option("--base", default="origin/main", help="Base ref the diff-scoped gates measure against.")
 @click.option(
@@ -27,12 +33,16 @@
     help="Run only these gates (repeatable). Default: every gate each root declares.",
 )
 @click.option("--dry-run", is_flag=True, help="Print the derived matrix without running it.")
-def cli(conventions: str, base: str, gates: tuple[str, ...], dry_run: bool) -> None:
-    with open(conventions, encoding="utf-8") as handle:
-        text = handle.read()
+def cli(conventions: tuple[str, ...], base: str, gates: tuple[str, ...], dry_run: bool) -> None:
+    try:
+        workflows = sources(conventions)
+    except NoGateMatrix as err:
+        click.echo(f"preflight: {err}", err=True)
+        raise SystemExit(1) from err
+    click.echo(f"preflight: gate matrix from {', '.join(path for path, _text in workflows)}")
     raise SystemExit(
         run(
-            text,
+            [text for _path, text in workflows],
             base,
             runner=default_runner,
             exists=os.path.exists,

internals/checks/src/checks/preflight/cli_test.py

@@ -1,6 +1,6 @@
 """Colocated unit tests for the preflight command (isolation -- no `CliRunner`).
 
-Driven through `.callback`; `run` and `open` are mocked at their import site.
+Driven through `.callback`; `sources` and `run` are mocked at their import site.
 """
 
 from unittest import mock
@@ -9,35 +9,82 @@
 
 from checks.preflight.cli import cli
 
+WORKFLOWS = [(".github/workflows/a-ci.yml", "jobs: {}"), (".github/workflows/b-ci.yml", "jobs: {}")]
 
-def invoke(gates=(), dry_run=False, **kwargs):
-    with mock.patch("checks.preflight.cli.open", mock.mock_open(read_data="jobs: {}")) as opened:
-        with mock.patch("checks.preflight.cli.run", **kwargs) as run:
-            with pytest.raises(SystemExit) as exc_info:
-                cli.callback(conventions="wf.yml", base="origin/x", gates=gates, dry_run=dry_run)
-    return opened, run, exc_info.value.code
 
+class NoGateMatrix(Exception):
+    """Stand-in for `matrix.NoGateMatrix` -- faked rather than imported.
 
-def test_reads_the_workflow_and_exits_with_runs_return_code():
-    opened, run, code = invoke(return_value=0)
-    opened.assert_called_once_with("wf.yml", encoding="utf-8")
-    assert run.call_args.args == ("jobs: {}", "origin/x")
+    Patched over the name the `except` clause resolves at raise time, so the
+    command's error path is exercised without importing the collaborator.
+    """
+
+
+def invoke(conventions=(), gates=(), dry_run=False, resolve=None, **kwargs):
+    with (
+        mock.patch("checks.preflight.cli.NoGateMatrix", NoGateMatrix),
+        mock.patch(
+            "checks.preflight.cli.sources", resolve or mock.Mock(return_value=WORKFLOWS)
+        ) as sources,
+        mock.patch("checks.preflight.cli.run", **kwargs) as run,
+        mock.patch("checks.preflight.cli.click.echo") as echo,
+    ):
+        with pytest.raises(SystemExit) as exc_info:
+            cli.callback(
+                conventions=conventions,
+                base="origin/x",
+                gates=gates,
+                dry_run=dry_run,
+            )
+    return sources, run, echo, exc_info.value.code
+
+
+def test_runs_every_resolved_workflow_and_exits_with_runs_return_code():
+    _sources, run, _echo, code = invoke(return_value=0)
+    assert run.call_args.args == (["jobs: {}", "jobs: {}"], "origin/x")
     assert code == 0
 
 
+def test_passes_the_named_workflows_through_to_the_resolver():
+    sources, _run, _echo, _code = invoke(conventions=("wf.yml",), return_value=0)
+    assert sources.call_args.args == (("wf.yml",),)
+
+
+def test_announces_which_workflows_the_matrix_came_from():
+    _sources, _run, echo, _code = invoke(return_value=0)
+    assert echo.call_args_list[0] == mock.call(
+        "preflight: gate matrix from .github/workflows/a-ci.yml, .github/workflows/b-ci.yml"
+    )
+
+
 def test_propagates_a_failing_matrix_as_exit_one():
-    assert invoke(return_value=1)[2] == 1
+    assert invoke(return_value=1)[3] == 1
+
+
+def test_reports_an_unresolvable_matrix_without_running_anything():
+    # #973: a workflow that no longer exists has to say so, not raise.
+    resolve = mock.Mock(side_effect=NoGateMatrix("--conventions gone.yml: no such workflow."))
+    _sources, run, echo, code = invoke(resolve=resolve, return_value=0)
+    assert (run.called, code) == (False, 1)
+    assert echo.call_args == mock.call(
+        "preflight: --conventions gone.yml: no such workflow.", err=True
+    )
 
 
 def test_forwards_the_gate_filter_and_dry_run_flag():
-    _opened, run, _code = invoke(gates=("unit-lint",), dry_run=True, return_value=0)
+    _sources, run, _echo, _code = invoke(gates=("unit-lint",), dry_run=True, return_value=0)
     assert run.call_args.kwargs["only"] == ("unit-lint",)
     assert run.call_args.kwargs["dry_run"] is True
 
 
-def test_defaults_to_the_ci_workflow_and_main():
-    conventions, base, gates, dry_run = cli.params
-    assert conventions.default == ".github/workflows/conventions.yml"
-    assert base.default == "origin/main"
-    assert gates.multiple is True
-    assert dry_run.is_flag is True
+def test_defaults_to_discovering_the_workflows_and_main():
+    # Parsed rather than read off the params: the effective default for
+    # `--conventions` is what #973 turned into a crash, and an empty one is what
+    # sends the command to discovery.
+    with cli.make_context("preflight", []) as ctx:
+        assert ctx.params == {
+            "conventions": (),
+            "base": "origin/main",
+            "gates": (),
+            "dry_run": False,
+        }

internals/checks/src/checks/preflight/gate.py

@@ -1,7 +1,7 @@
 """Run every (root, language, gate) pair CI declares, and report real exit codes (#781).
 
 Two failure modes this exists to remove: a hand-maintained pair list that drifts
-from `conventions.yml`, and hand-transcribed shell one-liners whose non-zero exit
+from what CI declares, and hand-transcribed shell one-liners whose non-zero exit
 gets masked (`... ; echo ok`). Here the argv is derived and the exit code is the
 return value.
 
@@ -153,7 +153,7 @@ def prepare(roots: list[Root], exists: Callable[[str], bool]) -> list[tuple[str,
 
 
 def run(
-    conventions: str,
+    workflows: Sequence[str],
     base: str,
     *,
     runner: Callable[[Sequence[str], str], int],
@@ -163,8 +163,12 @@ def run(
     only: Sequence[str] = (),
     dry_run: bool = False,
 ) -> int:
-    """Run the whole matrix; return 0 only when every pair passed."""
-    roots = parse_gate_matrix(conventions)
+    """Run the whole matrix; return 0 only when every pair passed.
+
+    `workflows` is the text of each workflow holding callers -- six of them post
+    #834, so the roots are the concatenation rather than one file's (#973).
+    """
+    roots = [root for text in workflows for root in parse_gate_matrix(text)]
     failures = []
     skipped = []
 

internals/checks/src/checks/preflight/gate_test.py

@@ -212,14 +212,14 @@ def it_keeps_going_past_a_non_python_root_to_the_ones_after_it():
         ]
 
 
-def drive(conventions=None, **kwargs):
+def drive(workflows=None, **kwargs):
     defaults = {
         "runner": lambda _argv, _cwd: 0,
         "exists": has_manifest,
         "e2e_config": lambda _config: {},
         "echo": lambda _line: None,
     }
-    return run(conventions or CONVENTIONS, "origin/main", **{**defaults, **kwargs})
+    return run(workflows or [CONVENTIONS], "origin/main", **{**defaults, **kwargs})
 
 
 def describe_run():
@@ -246,7 +246,7 @@ def it_counts_any_non_zero_exit_as_a_failure_including_a_negative_one():
     def it_skips_an_artifact_gate_without_failing_and_says_so():
         lines = []
         code = drive(
-            conventions=ARTIFACT_FIRST,
+            workflows=[ARTIFACT_FIRST],
             runner=lambda _argv, _cwd: 0,
             echo=lines.append,
         )
@@ -260,7 +260,7 @@ def it_skips_an_artifact_gate_without_failing_and_says_so():
     def it_keeps_going_past_a_skipped_gate_to_the_pairs_after_it():
         lines = []
         drive(
-            conventions=ARTIFACT_FIRST,
+            workflows=[ARTIFACT_FIRST],
             only=["packaging", "unit-lint"],
             echo=lines.append,
         )
@@ -305,11 +305,30 @@ def it_prints_every_pair_without_running_any_when_dry_run():
         assert (calls, code) == ([], 0)
         assert len([line for line in lines if line.startswith("==>")]) == 4
 
-    def it_takes_conventions_and_base_by_keyword():
+    def it_runs_the_pairs_of_every_workflow_it_is_given():
+        # Post-#834 the callers live in six workflows, so a matrix built from the
+        # first one alone would be a green run covering one lane (#973).
+        lines = []
+        drive(
+            workflows=[CONVENTIONS, CONVENTIONS.replace("python-sdk", "internals-checks")],
+            only=["unit-lint"],
+            echo=lines.append,
+        )
+        assert [line.split(": ")[0] for line in lines if line.startswith("==>")] == [
+            "==> python-sdk [python] unit-lint",
+            "==> internals-checks [python] unit-lint",
+        ]
+
+    def it_derives_no_pair_from_a_workflow_with_no_callers():
+        lines = []
+        assert drive(workflows=["jobs: {}"], echo=lines.append) == 0
+        assert lines == ["preflight: 0 failing pair(s), 0 skipped"]
+
+    def it_takes_the_workflows_and_base_by_keyword():
         # `*` (not `/`) before the injected seams: the two leading parameters must
-        # stay nameable, since every caller passes the workflow text by name.
+        # stay nameable, since every caller passes the workflow texts by name.
         assert run(
-            conventions=CONVENTIONS,
+            workflows=[CONVENTIONS],
             base="origin/main",
             runner=lambda _argv, _cwd: 0,
             exists=has_manifest,