diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f401cffe2..4fef06eec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -548,6 +548,28 @@ jobs: # lease, no torch, no numpy, no checkpoint. run: | python3 tests/scripts/test_ltx2_latent_dump.py + - name: The committed LTX-2 oracle goldens still hash to what records them (#2553) + # Registered in TWO places, like the suites above: here and in + # `scripts/agent-preflight.sh`. It was in ONE of them. `SUITES` held it + # from the start and this file never did, so the digests that anchor the + # oracle evidence gated on a developer box and were green on the forge. + # + # #2520 made that expensive. The suite now also covers + # `tests/parity/goldens/ltx2_oracle_latent/` -- the Phase A latents a + # future cross-decode reads INSTEAD of taking a GPU lease, 68 KB whose + # whole value is that they are the bytes the leased run produced. Without + # this step a pull request could overwrite `video_latent_bfloat16.raw`, + # or rewrite `SHA256SUMS` to match a corrupted one, and merge green. + # + # Nothing here can skip: standard library only -- `hashlib`, `json`, + # `re`, `pathlib` -- over four committed files, no numpy, no build, no + # GPU, no lease, no checkpoint, no network, 0.02 s. The step needs no + # guard, and a guard is what would turn this absence into false coverage. + # The registration itself is pinned in `REQUIRED_SUITE_REGISTRATIONS`, + # so deleting this step reds `check-test-registration.py` rather than + # restoring the gap. + run: | + python3 tests/scripts/test_ltx2_oracle_goldens.py - name: Every LTX-2.5 A/B arm sets a knob value the dispatch parses (#1751) # The `VLLM_LTX2_DIT_FLASH_ATTN` values a harness exports and the values # `ltx2_device.cpp` matches are two literals in two languages, and nothing diff --git a/scripts/check-test-registration.py b/scripts/check-test-registration.py index 729707ac6..6fc563111 100644 --- a/scripts/check-test-registration.py +++ b/scripts/check-test-registration.py @@ -79,6 +79,17 @@ class the labelled gate itself exists to close. ``REQUIRED_LABEL_SELECTIONS`` # the M9-M12 mutation cases match on. REQUIRED_SUITE_REGISTRATIONS = { "test_sglang_lease_identity": "tests/scripts/test_sglang_lease_identity.py", + # #2553. Added the moment the suite reached both lanes, which is the only + # moment it costs nothing: it was preflight-only, so the CI step landing + # beside this line would otherwise have been deletable at rc=0 -- the exact + # #1833 finding this set exists to answer. It recomputes the sha256 of the + # #1864 reference render and the #2514 Phase A latents, the evidence + # `.agents/oracles/ltx-2.md` cites for `gateable = yes`. + # + # This is the widening the seeding comment above anticipates, one classified + # suite at a time. It is NOT the symmetric population rule, which still + # belongs to #408 and #1509. + "test_ltx2_oracle_goldens": "tests/scripts/test_ltx2_oracle_goldens.py", } def _without_line_comments(text: str) -> str: diff --git a/tests/scripts/test_check_test_registration.py b/tests/scripts/test_check_test_registration.py index 30128e56b..36d7587f0 100644 --- a/tests/scripts/test_check_test_registration.py +++ b/tests/scripts/test_check_test_registration.py @@ -50,6 +50,9 @@ - name: The SGLang lease identity gate detects a tree that is not the pin run: | python3 tests/scripts/test_sglang_lease_identity.py + - name: The committed LTX-2 oracle goldens still hash to what records them + run: | + python3 tests/scripts/test_ltx2_oracle_goldens.py """ PASSING_PREFLIGHT = """CHECKERS=( @@ -58,6 +61,7 @@ SUITES=( test_check_test_registration test_sglang_lease_identity + test_ltx2_oracle_goldens ) for checker in "${CHECKERS[@]}"; do python3 "scripts/$checker.py" @@ -331,6 +335,42 @@ def test_M57_deleting_the_pinned_ci_suite_fails(self) -> None: "test_sglang_lease_identity is missing from the CI suite lane", ) + # The SECOND pinned suite (#2553). These two cases are what makes widening + # `REQUIRED_SUITE_REGISTRATIONS` a semantic change rather than a fixture + # edit. Adding the suite to `PASSING_CI` and `PASSING_PREFLIGHT` alone is + # symmetric and additive, so the base checker -- which pins only + # `test_sglang_lease_identity` -- stays green on it and the widening carries + # no evidence at all; `check-pr-size.py` rejected exactly that, because a + # test whose only job is to KEEP an existing case green cannot fail for the + # intended reason. Deleting one lane's registration must name THIS suite. + # + # They assert the checker's output on a mutated input, never + # `"test_ltx2_oracle_goldens" in mod.REQUIRED_SUITE_REGISTRATIONS`, which + # would be a tautology over the constant it is supposed to hold. And they + # are deliberately not named `test_M*`: `mutation_suite_integrity_errors` + # requires the `test_M*` set to equal the digest-pinned manifest exactly, so + # an M-number here would drag `check_test_registration_mutations.txt` and + # `MUTATION_MANIFEST_SHA256` into a change that is not about the manifest. + def test_pinned_ltx2_goldens_suite_absent_from_ci_is_named(self) -> None: + mutated = self.ci.replace( + " python3 tests/scripts/test_ltx2_oracle_goldens.py\n", "" + ) + self.assertNotEqual(mutated, self.ci) + self.assert_wiring_error( + self.preflight, + mutated, + "test_ltx2_oracle_goldens is missing from the CI suite lane", + ) + + def test_pinned_ltx2_goldens_suite_absent_from_preflight_is_named(self) -> None: + mutated = self.preflight.replace(" test_ltx2_oracle_goldens\n", "") + self.assertNotEqual(mutated, self.preflight) + self.assert_wiring_error( + mutated, + self.ci, + "test_ltx2_oracle_goldens is missing from preflight SUITES", + ) + def test_M18_ci_commands_behind_false_shell_branch_fail(self) -> None: mutated = self.ci.replace( " python3 scripts/check-test-registration.py\n"