|
| 1 | +"""Run the pinned ACP TCK against the current checkout's v1 and v2 agent APIs.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import argparse |
| 6 | +import json |
| 7 | +import shutil |
| 8 | +import subprocess |
| 9 | +import sys |
| 10 | +from pathlib import Path |
| 11 | +from typing import Any |
| 12 | + |
| 13 | +ROOT = Path(__file__).resolve().parents[1] |
| 14 | +TCK_REVISIONS = { |
| 15 | + 1: "b7f40fb3c003c45926547be5a1f7e2b374035815", |
| 16 | + 2: "fa00ccdc13dfaa4b288a91877c1f3e468af4aa7d", |
| 17 | +} |
| 18 | +TCK_SCHEMA_REVISIONS = { |
| 19 | + 1: "beac4ba96190018e9f6d82e148ff275084c5aa3e", |
| 20 | + 2: "9af0e9f748db9f4cc4c410a7b212ead4f98ae78c", |
| 21 | +} |
| 22 | +CANCEL_PROMPT = "wait_for_cancel" |
| 23 | + |
| 24 | + |
| 25 | +def executable(name: str) -> str: |
| 26 | + path = shutil.which(name) |
| 27 | + if path is None: |
| 28 | + raise RuntimeError(f"{name} is not on PATH") |
| 29 | + return path |
| 30 | + |
| 31 | + |
| 32 | +def git_revision(path: Path) -> str: |
| 33 | + return subprocess.check_output( # noqa: S603 |
| 34 | + [executable("git"), "-C", str(path), "rev-parse", "HEAD"], text=True |
| 35 | + ).strip() |
| 36 | + |
| 37 | + |
| 38 | +def sdk_python() -> Path: |
| 39 | + python = ROOT / ".venv" / "bin" / "python" |
| 40 | + if not python.exists(): |
| 41 | + raise RuntimeError("SDK virtual environment is missing; run uv sync --frozen first") |
| 42 | + module = subprocess.check_output( # noqa: S603 |
| 43 | + [str(python), "-c", "import acp; print(acp.__file__)"], text=True |
| 44 | + ).strip() |
| 45 | + expected = ROOT / "src" / "acp" / "__init__.py" |
| 46 | + if Path(module).resolve() != expected.resolve(): |
| 47 | + raise RuntimeError(f"Agent Python imports {module}, expected {expected}") |
| 48 | + return python |
| 49 | + |
| 50 | + |
| 51 | +def run_suite(tck_dir: Path, output_dir: Path, python: Path, version: int, timeout: int) -> int: |
| 52 | + report = output_dir / f"v{version}.json" |
| 53 | + log = output_dir / f"v{version}.log" |
| 54 | + report.unlink(missing_ok=True) |
| 55 | + command = [ |
| 56 | + executable("uv"), |
| 57 | + "--project", |
| 58 | + str(tck_dir), |
| 59 | + "run", |
| 60 | + "--locked", |
| 61 | + "acp-tck", |
| 62 | + "--protocol-version", |
| 63 | + str(version), |
| 64 | + "--cancel-prompt", |
| 65 | + CANCEL_PROMPT, |
| 66 | + "--timeout", |
| 67 | + str(timeout), |
| 68 | + "--test-timeout", |
| 69 | + str(max(45, timeout * 4)), |
| 70 | + "--report-json", |
| 71 | + str(report), |
| 72 | + "--", |
| 73 | + str(python), |
| 74 | + str(ROOT / "tests" / "tck" / f"agent_v{version}.py"), |
| 75 | + ] |
| 76 | + print(f"Running ACP v{version} TCK; detailed output: {log}", flush=True) |
| 77 | + with log.open("w") as output: |
| 78 | + return subprocess.run( # noqa: S603 |
| 79 | + command, cwd=ROOT, check=False, stdout=output, stderr=subprocess.STDOUT |
| 80 | + ).returncode |
| 81 | + |
| 82 | + |
| 83 | +def check_mandatory(requirements: list[dict[str, Any]], version: int) -> list[str]: |
| 84 | + errors = [] |
| 85 | + for item in requirements: |
| 86 | + if item.get("tier") != "MANDATORY": |
| 87 | + continue |
| 88 | + if item.get("status") != "PASS": |
| 89 | + errors.append(f"v{version}: {item['id']} is {item.get('status')}, expected PASS") |
| 90 | + for test in item.get("tests", []): |
| 91 | + if test.get("status") != "PASS": |
| 92 | + errors.append(f"v{version}: {test.get('nodeid')} is {test.get('status')}, expected PASS") |
| 93 | + return errors |
| 94 | + |
| 95 | + |
| 96 | +def check_provenance(report: dict[str, Any], version: int, python: Path) -> list[str]: |
| 97 | + errors = [] |
| 98 | + if report.get("protocol_version") != version: |
| 99 | + errors.append(f"v{version}: report protocol_version is {report.get('protocol_version')!r}") |
| 100 | + if report.get("schema_revision") != TCK_SCHEMA_REVISIONS[version]: |
| 101 | + errors.append(f"v{version}: unexpected TCK schema revision {report.get('schema_revision')!r}") |
| 102 | + command = report.get("agent_command", []) |
| 103 | + if not command or Path(command[0]).resolve() != python.resolve(): |
| 104 | + errors.append(f"v{version}: report does not identify the current checkout's Python") |
| 105 | + expected_probe = ROOT / "tests" / "tck" / f"agent_v{version}.py" |
| 106 | + if len(command) < 2 or Path(command[1]).resolve() != expected_probe.resolve(): |
| 107 | + errors.append(f"v{version}: report does not identify the v{version} probe") |
| 108 | + return errors |
| 109 | + |
| 110 | + |
| 111 | +def inspect_report( |
| 112 | + path: Path, version: int, *, strict: bool, tck_exit: int, python: Path |
| 113 | +) -> tuple[list[str], dict[str, Any]]: |
| 114 | + if not path.exists(): |
| 115 | + return [f"v{version}: TCK wrote no JSON report (exit {tck_exit})"], {} |
| 116 | + try: |
| 117 | + report = json.loads(path.read_text()) |
| 118 | + except (OSError, json.JSONDecodeError) as exc: |
| 119 | + return [f"v{version}: cannot read TCK report: {exc}"], {} |
| 120 | + |
| 121 | + errors = check_provenance(report, version, python) |
| 122 | + verdict = report.get("verdict", {}) |
| 123 | + for flag in ("blocked_by_auth", "blocked_by_version_mismatch"): |
| 124 | + if verdict.get(flag): |
| 125 | + errors.append(f"v{version}: {flag}") |
| 126 | + |
| 127 | + requirements = report.get("requirements", []) |
| 128 | + if not isinstance(requirements, list) or not requirements: |
| 129 | + errors.append(f"v{version}: report contains no requirements") |
| 130 | + requirements = [] |
| 131 | + failures = [f"{item['id']} ({item['tier']})" for item in requirements if item.get("status") == "FAIL"] |
| 132 | + print( |
| 133 | + f"v{version}: TCK exit={tck_exit}, conformant={verdict.get('conformant')}, " |
| 134 | + f"failures={', '.join(failures) or 'none'}, schema={report.get('schema_revision')}", |
| 135 | + flush=True, |
| 136 | + ) |
| 137 | + |
| 138 | + if strict: |
| 139 | + if tck_exit != 0 or verdict.get("conformant") is not True: |
| 140 | + errors.append(f"v{version}: TCK conformance verdict failed") |
| 141 | + errors.extend(check_mandatory(requirements, version)) |
| 142 | + return errors, report |
| 143 | + |
| 144 | + |
| 145 | +def main() -> int: |
| 146 | + parser = argparse.ArgumentParser(description=__doc__) |
| 147 | + parser.add_argument("--tck-v1-dir", type=Path, required=True, help="checkout of the pinned v1 TCK commit") |
| 148 | + parser.add_argument("--tck-v2-dir", type=Path, required=True, help="checkout of the alpha.5 v2 TCK commit") |
| 149 | + parser.add_argument("--output-dir", type=Path, default=ROOT / "build" / "tck") |
| 150 | + parser.add_argument("--timeout", type=int, default=10, help="TCK per-response timeout in seconds") |
| 151 | + args = parser.parse_args() |
| 152 | + tck_dirs = {1: args.tck_v1_dir.resolve(), 2: args.tck_v2_dir.resolve()} |
| 153 | + output_dir = args.output_dir.resolve() |
| 154 | + |
| 155 | + for version, tck_dir in tck_dirs.items(): |
| 156 | + if git_revision(tck_dir) != TCK_REVISIONS[version]: |
| 157 | + parser.error(f"v{version} TCK must be checked out at {TCK_REVISIONS[version]}") |
| 158 | + python = sdk_python() |
| 159 | + output_dir.mkdir(parents=True, exist_ok=True) |
| 160 | + |
| 161 | + errors: list[str] = [] |
| 162 | + summaries: dict[str, Any] = {} |
| 163 | + for version in (1, 2): |
| 164 | + tck_dir = tck_dirs[version] |
| 165 | + exit_code = run_suite(tck_dir, output_dir, python, version, args.timeout) |
| 166 | + problems, report = inspect_report( |
| 167 | + output_dir / f"v{version}.json", |
| 168 | + version, |
| 169 | + strict=version == 1, |
| 170 | + tck_exit=exit_code, |
| 171 | + python=python, |
| 172 | + ) |
| 173 | + errors.extend(problems) |
| 174 | + summaries[f"v{version}"] = { |
| 175 | + "tck_revision": TCK_REVISIONS[version], |
| 176 | + "tck_exit": exit_code, |
| 177 | + "tck_schema_revision": report.get("schema_revision"), |
| 178 | + "verdict": report.get("verdict"), |
| 179 | + } |
| 180 | + |
| 181 | + manifest = { |
| 182 | + "sdk_revision": git_revision(ROOT), |
| 183 | + "sdk_schema_refs": { |
| 184 | + "v1": (ROOT / "schema" / "VERSION").read_text().strip(), |
| 185 | + "v2": (ROOT / "schema" / "v2" / "VERSION").read_text().strip(), |
| 186 | + }, |
| 187 | + "sdk_python": str(python.resolve()), |
| 188 | + "sdk_python_version": subprocess.check_output([str(python), "--version"], text=True).strip(), # noqa: S603 |
| 189 | + "tck_python_versions": { |
| 190 | + f"v{version}": subprocess.check_output( # noqa: S603 |
| 191 | + [str(tck_dir / ".venv" / "bin" / "python"), "--version"], text=True |
| 192 | + ).strip() |
| 193 | + for version, tck_dir in tck_dirs.items() |
| 194 | + }, |
| 195 | + "runs": summaries, |
| 196 | + } |
| 197 | + (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") |
| 198 | + for error in errors: |
| 199 | + print(error, file=sys.stderr) |
| 200 | + return 1 if errors else 0 |
| 201 | + |
| 202 | + |
| 203 | +if __name__ == "__main__": |
| 204 | + sys.exit(main()) |
0 commit comments