Skip to content

Commit db456be

Browse files
committed
ci: gate mandatory ACP v2 TCK behavior
Gate MANDATORY outcomes in the v2 lane, not just the v1 lane, so a v2 FAIL or NOT_TESTED now fails CI. The v1 lane still additionally requires the TCK's own conformance verdict; the v2 lane does not gate that verdict and never gates ADVISORY results. Accept only the two conditional auth skips alpha.5 reports for an agent that advertises no auth methods (ACP-AUTH-206/207). A skip counts only when the requirement is SKIPPED, the fixture reports no auth capability, and the node id and skip reason match the pinned text exactly; anything else stays gated.
1 parent c9adecc commit db456be

3 files changed

Lines changed: 185 additions & 9 deletions

File tree

‎docs/tck.md‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,16 @@ environment and lockfile; it never supplies the agent's SDK package. It writes `
3636
`build/tck/v2.json`, `build/tck/manifest.json`, and a detailed log for each suite.
3737

3838
The v1 lane is a CI gate: the official TCK verdict and every mandatory test outcome must pass.
39-
This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip. The
40-
v2 lane always runs and reports its current behavior. Even with the alpha.5 TCK, the SDK
41-
currently fails mandatory version-negotiation and batch-response checks, so v2 is not yet a CI
42-
gate. Its failures remain visible in the report. Promote it after those SDK behaviors are fixed.
39+
This catches cancellation tests that TCK's aggregate verdict would otherwise allow to skip.
40+
41+
The v2 lane gates mandatory outcomes only: any `MANDATORY` `FAIL`, `NOT_TESTED`, or unexpected
42+
`SKIPPED` requirement fails CI. The two conditional auth checks `ACP-AUTH-206` and `ACP-AUTH-207`
43+
are the sole exception — alpha.5 skips them when the agent advertises no auth methods, and the
44+
runner accepts those skips only when the fixture reports no `auth` capability and each skip's
45+
node id and reason match the pinned text. The v2 lane does not gate the TCK's own verdict (which
46+
also folds in `CAPABILITY` failures) and never gates `ADVISORY` results, which stay as advisory
47+
reporting.
48+
4349
The CI job uploads all reports and logs even when its gate fails.
4450

4551
The probes advertise only the capabilities they implement. Optional capability tests that remain

‎scripts/run_tck.py‎

Lines changed: 70 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from __future__ import annotations
44

55
import argparse
6+
import ast
67
import json
78
import shutil
89
import subprocess
@@ -21,6 +22,20 @@
2122
}
2223
CANCEL_PROMPT = "wait_for_cancel"
2324

25+
# The only v2 MANDATORY SKIPs the gate accepts: alpha.5's two conditional auth checks, which have
26+
# nothing to inspect unless the agent advertises auth methods. Pinned per requirement id to the test
27+
# that must skip and its exact reason, so any other MANDATORY SKIP fails CI.
28+
V2_PINNED_SKIPS = {
29+
"ACP-AUTH-206": (
30+
"test_authentication.py::test_auth_method_type_is_a_defined_or_prefixed_value",
31+
"Skipped: agent advertises no authMethods",
32+
),
33+
"ACP-AUTH-207": (
34+
"test_authentication.py::test_terminal_auth_method_descriptor_shape",
35+
"Skipped: agent advertises no type=='terminal' authMethods entry; nothing to check",
36+
),
37+
}
38+
2439

2540
def executable(name: str) -> str:
2641
path = shutil.which(name)
@@ -80,11 +95,60 @@ def run_suite(tck_dir: Path, output_dir: Path, python: Path, version: int, timeo
8095
).returncode
8196

8297

83-
def check_mandatory(requirements: list[dict[str, Any]], version: int) -> list[str]:
98+
def advertises_auth(capabilities: Any) -> bool:
99+
"""Whether the report's recorded initialize capabilities advertise agent-side auth support.
100+
101+
Fail-closed: capabilities that are missing or not an object count as advertising auth, so an
102+
unverifiable fixture can never unlock the pinned skip.
103+
"""
104+
return not (isinstance(capabilities, dict) and capabilities.get("auth") is None)
105+
106+
107+
def skip_reason(message: Any) -> str | None:
108+
"""The `Skipped: ...` reason inside a TCK test message.
109+
110+
alpha.5 records a message as the repr of `(file, line, reason)`; fall back to the raw string
111+
so an unrecognised shape can never accidentally match a pinned reason.
112+
"""
113+
if not isinstance(message, str) or not message:
114+
return None
115+
try:
116+
parsed = ast.literal_eval(message)
117+
except (SyntaxError, ValueError):
118+
return message
119+
if isinstance(parsed, tuple) and parsed and isinstance(parsed[-1], str):
120+
return parsed[-1]
121+
return message
122+
123+
124+
def pinned_auth_skip(requirement: dict[str, Any], version: int, report: dict[str, Any]) -> bool:
125+
"""True if this MANDATORY SKIP is one of the two pinned alpha.5 conditional auth checks.
126+
127+
Requires all four conditions: the v2 lane, a requirement the TCK itself marked SKIPPED, a
128+
fixture that advertises no auth, and a test whose node id and skip reason match the pin
129+
exactly.
130+
"""
131+
pinned = V2_PINNED_SKIPS.get(str(requirement.get("id"))) if version == 2 else None
132+
if pinned is None or requirement.get("status") != "SKIPPED":
133+
return False
134+
if advertises_auth(report.get("agent_capabilities")):
135+
return False
136+
nodeid, reason = pinned
137+
tests = requirement.get("tests", [])
138+
return bool(tests) and all(
139+
test.get("status") == "SKIPPED" and test.get("nodeid") == nodeid and skip_reason(test.get("message")) == reason
140+
for test in tests
141+
)
142+
143+
144+
def check_mandatory(requirements: list[dict[str, Any]], version: int, report: dict[str, Any]) -> list[str]:
84145
errors = []
85146
for item in requirements:
86147
if item.get("tier") != "MANDATORY":
87148
continue
149+
if pinned_auth_skip(item, version, report):
150+
print(f"v{version}: {item['id']} is SKIPPED (pinned conditional auth check)", flush=True)
151+
continue
88152
if item.get("status") != "PASS":
89153
errors.append(f"v{version}: {item['id']} is {item.get('status')}, expected PASS")
90154
for test in item.get("tests", []):
@@ -135,10 +199,11 @@ def inspect_report(
135199
flush=True,
136200
)
137201

138-
if strict:
139-
if tck_exit != 0 or verdict.get("conformant") is not True:
140-
errors.append(f"v{version}: TCK conformance verdict failed")
141-
errors.extend(check_mandatory(requirements, version))
202+
if strict and (tck_exit != 0 or verdict.get("conformant") is not True):
203+
errors.append(f"v{version}: TCK conformance verdict failed")
204+
# Both lanes gate MANDATORY outcomes; only the v1 lane additionally requires the TCK's own
205+
# verdict, which also folds in CAPABILITY failures. ADVISORY results are never gated.
206+
errors.extend(check_mandatory(requirements, version, report))
142207
return errors, report
143208

144209

‎tests/test_run_tck_gate.py‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
"""The v2 MANDATORY gate in `scripts/run_tck.py`, and its two pinned alpha.5 auth skips.
2+
3+
`scripts/run_tck.py` drives an external TCK checkout, so these tests call `check_mandatory`
4+
directly with synthetic requirement rows instead of running a suite.
5+
"""
6+
7+
from __future__ import annotations
8+
9+
from typing import Any
10+
11+
import pytest
12+
13+
from scripts.run_tck import V2_PINNED_SKIPS, check_mandatory
14+
15+
NO_AUTH_CAPABILITIES = {"session": {}}
16+
17+
18+
def skipped(requirement_id: str, *, nodeid: str, reason: str) -> dict[str, Any]:
19+
return {
20+
"id": requirement_id,
21+
"tier": "MANDATORY",
22+
"status": "SKIPPED",
23+
"tests": [{"nodeid": nodeid, "status": "SKIPPED", "message": repr(("/tck/test_authentication.py", 1, reason))}],
24+
}
25+
26+
27+
def pinned_row(requirement_id: str) -> dict[str, Any]:
28+
nodeid, reason = V2_PINNED_SKIPS[requirement_id]
29+
return skipped(requirement_id, nodeid=nodeid, reason=reason)
30+
31+
32+
def report(capabilities: Any = NO_AUTH_CAPABILITIES) -> dict[str, Any]:
33+
return {"agent_capabilities": capabilities}
34+
35+
36+
def test_both_pinned_auth_skips_are_accepted_for_an_unauthenticated_v2_fixture() -> None:
37+
rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")]
38+
assert check_mandatory(rows, 2, report()) == []
39+
40+
41+
@pytest.mark.parametrize("requirement_id", ["ACP-AUTH-206", "ACP-AUTH-207"])
42+
def test_pin_requires_the_exact_skip_reason(requirement_id: str) -> None:
43+
nodeid, reason = V2_PINNED_SKIPS[requirement_id]
44+
row = skipped(requirement_id, nodeid=nodeid, reason=f"{reason} -- but for a different reason")
45+
assert check_mandatory([row], 2, report()) != []
46+
47+
48+
@pytest.mark.parametrize("requirement_id", ["ACP-AUTH-206", "ACP-AUTH-207"])
49+
def test_pin_requires_a_fixture_that_advertises_no_auth(requirement_id: str) -> None:
50+
row = pinned_row(requirement_id)
51+
assert check_mandatory([row], 2, report({"session": {}, "auth": {"terminal": {}}})) != []
52+
53+
54+
@pytest.mark.parametrize("capabilities", [{}, {"auth": None}])
55+
def test_pin_accepts_capabilities_that_report_no_auth(capabilities: Any) -> None:
56+
rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")]
57+
assert check_mandatory(rows, 2, report(capabilities)) == []
58+
59+
60+
@pytest.mark.parametrize(
61+
"capabilities",
62+
[None, [], "unexpected"],
63+
ids=["missing", "wrong-type", "string"],
64+
)
65+
def test_pin_fails_closed_on_unverifiable_capabilities(capabilities: Any) -> None:
66+
rows = [pinned_row("ACP-AUTH-206"), pinned_row("ACP-AUTH-207")]
67+
assert check_mandatory(rows, 2, report(capabilities)) != []
68+
69+
70+
def test_unpinned_mandatory_skip_still_fails_v2() -> None:
71+
row = skipped("ACP-AUTH-202", nodeid="test_authentication.py::test_other", reason="Skipped: anything")
72+
assert check_mandatory([row], 2, report()) != []
73+
74+
75+
def test_pinned_skip_fails_on_the_v1_lane() -> None:
76+
assert check_mandatory([pinned_row("ACP-AUTH-206")], 1, report()) != []
77+
78+
79+
def test_fail_row_with_pinned_id_reason_and_nodeid_is_rejected() -> None:
80+
"""A FAIL row carrying a pinned id, node id, and reason must not unlock the gate.
81+
82+
`pinned_auth_skip` also requires the requirement's own status to be SKIPPED, so a corrupted
83+
row whose tests all claim SKIPPED but whose aggregate verdict is FAIL stays gated.
84+
"""
85+
requirement_id = "ACP-AUTH-206"
86+
nodeid, reason = V2_PINNED_SKIPS[requirement_id]
87+
row = pinned_row(requirement_id) | {"status": "FAIL"}
88+
errors = check_mandatory([row], 2, report())
89+
assert f"v2: {requirement_id} is FAIL, expected PASS" in errors
90+
assert f"v2: {nodeid} is SKIPPED, expected PASS" in errors
91+
92+
93+
def test_mandatory_fail_and_not_tested_still_fail_v2() -> None:
94+
failed = {"id": "ACP-BATCH-201", "tier": "MANDATORY", "status": "FAIL", "tests": []}
95+
not_tested = {"id": "ACP-INIT-201", "tier": "MANDATORY", "status": "NOT_TESTED", "tests": []}
96+
assert check_mandatory([failed, not_tested], 2, report()) != []
97+
98+
99+
def test_advisory_and_other_tiers_are_never_gated() -> None:
100+
rows = [
101+
{"id": "ACP-AUTH-201", "tier": "ADVISORY", "status": "FAIL", "tests": []},
102+
{"id": "ACP-SESSION-004", "tier": "ADVISORY", "status": "SKIPPED", "tests": []},
103+
{"id": "ACP-AUTH-203", "tier": "CAPABILITY", "status": "SKIPPED", "tests": []},
104+
]
105+
assert check_mandatory(rows, 2, report()) == []

0 commit comments

Comments
 (0)