|
| 1 | +"""The v2 MANDATORY gate in `scripts/run_tck.py`, and its two pinned alpha.5 auth skips. |
| 2 | +
|
| 3 | +`scripts/run_tck.py` drives an external TCK checkout, so these tests call `check_mandatory` |
| 4 | +directly with synthetic requirement rows instead of running a suite. |
| 5 | +""" |
| 6 | + |
| 7 | +from __future__ import annotations |
| 8 | + |
| 9 | +from typing import Any |
| 10 | + |
| 11 | +import pytest |
| 12 | + |
| 13 | +from scripts.run_tck import V2_PINNED_SKIPS, check_mandatory |
| 14 | + |
| 15 | +NO_AUTH_CAPABILITIES = {"session": {}} |
| 16 | + |
| 17 | + |
| 18 | +def skipped(requirement_id: str, *, nodeid: str, reason: str) -> dict[str, Any]: |
| 19 | + return { |
| 20 | + "id": requirement_id, |
| 21 | + "tier": "MANDATORY", |
| 22 | + "status": "SKIPPED", |
| 23 | + "tests": [{"nodeid": nodeid, "status": "SKIPPED", "message": repr(("/tck/test_authentication.py", 1, reason))}], |
| 24 | + } |
| 25 | + |
| 26 | + |
| 27 | +def pinned_row(requirement_id: str) -> dict[str, Any]: |
| 28 | + nodeid, reason = V2_PINNED_SKIPS[requirement_id] |
| 29 | + return skipped(requirement_id, nodeid=nodeid, reason=reason) |
| 30 | + |
| 31 | + |
| 32 | +def report(capabilities: Any = NO_AUTH_CAPABILITIES) -> dict[str, Any]: |
| 33 | + return {"agent_capabilities": capabilities} |
| 34 | + |
| 35 | + |
| 36 | +def test_both_pinned_auth_skips_are_accepted_for_an_unauthenticated_v2_fixture() -> None: |
| 37 | + rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")] |
| 38 | + assert check_mandatory(rows, 2, report()) == [] |
| 39 | + |
| 40 | + |
| 41 | +@pytest.mark.parametrize("requirement_id", ["ACP-AUTH-206", "ACP-AUTH-207"]) |
| 42 | +def test_pin_requires_the_exact_skip_reason(requirement_id: str) -> None: |
| 43 | + nodeid, reason = V2_PINNED_SKIPS[requirement_id] |
| 44 | + row = skipped(requirement_id, nodeid=nodeid, reason=f"{reason} -- but for a different reason") |
| 45 | + assert check_mandatory([row], 2, report()) != [] |
| 46 | + |
| 47 | + |
| 48 | +@pytest.mark.parametrize("requirement_id", ["ACP-AUTH-206", "ACP-AUTH-207"]) |
| 49 | +def test_pin_requires_a_fixture_that_advertises_no_auth(requirement_id: str) -> None: |
| 50 | + row = pinned_row(requirement_id) |
| 51 | + assert check_mandatory([row], 2, report({"session": {}, "auth": {"terminal": {}}})) != [] |
| 52 | + |
| 53 | + |
| 54 | +@pytest.mark.parametrize("capabilities", [{}, {"auth": None}]) |
| 55 | +def test_pin_accepts_capabilities_that_report_no_auth(capabilities: Any) -> None: |
| 56 | + rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")] |
| 57 | + assert check_mandatory(rows, 2, report(capabilities)) == [] |
| 58 | + |
| 59 | + |
| 60 | +@pytest.mark.parametrize( |
| 61 | + "capabilities", |
| 62 | + [None, [], "unexpected"], |
| 63 | + ids=["missing", "wrong-type", "string"], |
| 64 | +) |
| 65 | +def test_pin_fails_closed_on_unverifiable_capabilities(capabilities: Any) -> None: |
| 66 | + rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")] |
| 67 | + assert check_mandatory(rows, 2, report(capabilities)) != [] |
| 68 | + |
| 69 | + |
| 70 | +def test_unpinned_mandatory_skip_still_fails_v2() -> None: |
| 71 | + row = skipped("ACP-AUTH-202", nodeid="test_authentication.py::test_other", reason="Skipped: anything") |
| 72 | + assert check_mandatory([row], 2, report()) != [] |
| 73 | + |
| 74 | + |
| 75 | +def test_pinned_skip_fails_on_the_v1_lane() -> None: |
| 76 | + assert check_mandatory([pinned_row("ACP-AUTH-206")], 1, report()) != [] |
| 77 | + |
| 78 | + |
| 79 | +def test_fail_row_with_pinned_id_reason_and_nodeid_is_rejected() -> None: |
| 80 | + """A FAIL row carrying a pinned id, node id, and reason must not unlock the gate. |
| 81 | +
|
| 82 | + `pinned_auth_skip` also requires the requirement's own status to be SKIPPED, so a corrupted |
| 83 | + row whose tests all claim SKIPPED but whose aggregate verdict is FAIL stays gated. |
| 84 | + """ |
| 85 | + requirement_id = "ACP-AUTH-206" |
| 86 | + nodeid, reason = V2_PINNED_SKIPS[requirement_id] |
| 87 | + row = pinned_row(requirement_id) | {"status": "FAIL"} |
| 88 | + errors = check_mandatory([row], 2, report()) |
| 89 | + assert f"v2: {requirement_id} is FAIL, expected PASS" in errors |
| 90 | + assert f"v2: {nodeid} is SKIPPED, expected PASS" in errors |
| 91 | + |
| 92 | + |
| 93 | +def test_mandatory_fail_and_not_tested_still_fail_v2() -> None: |
| 94 | + failed = {"id": "ACP-BATCH-201", "tier": "MANDATORY", "status": "FAIL", "tests": []} |
| 95 | + not_tested = {"id": "ACP-INIT-201", "tier": "MANDATORY", "status": "NOT_TESTED", "tests": []} |
| 96 | + assert check_mandatory([failed, not_tested], 2, report()) != [] |
| 97 | + |
| 98 | + |
| 99 | +def test_advisory_and_other_tiers_are_never_gated() -> None: |
| 100 | + rows = [ |
| 101 | + {"id": "ACP-AUTH-201", "tier": "ADVISORY", "status": "FAIL", "tests": []}, |
| 102 | + {"id": "ACP-SESSION-004", "tier": "ADVISORY", "status": "SKIPPED", "tests": []}, |
| 103 | + {"id": "ACP-AUTH-203", "tier": "CAPABILITY", "status": "SKIPPED", "tests": []}, |
| 104 | + ] |
| 105 | + assert check_mandatory(rows, 2, report()) == [] |
0 commit comments