diff --git a/.github/workflows/chassis-shootout-contract.yml b/.github/workflows/chassis-shootout-contract.yml new file mode 100644 index 00000000..060eaacc --- /dev/null +++ b/.github/workflows/chassis-shootout-contract.yml @@ -0,0 +1,130 @@ +name: Chassis shootout runner contract + +on: + workflow_dispatch: + push: + paths: + - ".github/workflows/chassis-shootout-contract.yml" + - "tools/run_chassis_ci_contract.py" + - "tests/test_chassis_ci_contract.py" + +permissions: + contents: read + +concurrency: + group: chassis-shootout-contract-${{ github.ref }}-${{ github.run_id }} + cancel-in-progress: false + +jobs: + contract-tests: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.13" + + - name: Run runner contract tests + run: python -m unittest -v tests.test_chassis_ci_contract + + deterministic-adapter-smoke: + needs: contract-tests + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + candidate: [nano, agentless] + env: + EVIDENCE_DIR: logs/chassis-shootout/contract/${{ matrix.candidate }} + WORKDIR: .tmp/chassis-shootout/${{ matrix.candidate }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.13" + + - name: Record runner identity + run: | + mkdir -p "$EVIDENCE_DIR" + git rev-parse HEAD | tee "$EVIDENCE_DIR/project-commit.txt" + python -VV | tee "$EVIDENCE_DIR/python.txt" + uname -a | tee "$EVIDENCE_DIR/uname.txt" + + - name: Run deterministic candidate fixture + run: | + python tools/run_chassis_ci_contract.py \ + --candidate "${{ matrix.candidate }}" \ + --candidate-revision "fixture-v1" \ + --candidate-command-json '["python","-c","from pathlib import Path; Path(\"result.txt\").write_text(\"fixed\\n\")"]' \ + --verifier-command-json '["python","-c","from pathlib import Path; raise SystemExit(0 if Path(\"result.txt\").read_text() == \"fixed\\n\" else 1)"]' \ + --workdir "$WORKDIR" \ + --output "$EVIDENCE_DIR/result.json" + + - name: Upload candidate evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: chassis-contract-${{ matrix.candidate }}-${{ github.sha }} + retention-days: 30 + if-no-files-found: error + path: logs/chassis-shootout/contract/${{ matrix.candidate }}/ + + negative-control: + needs: contract-tests + runs-on: ubuntu-24.04 + timeout-minutes: 15 + env: + EVIDENCE_DIR: logs/chassis-shootout/contract/broken + WORKDIR: .tmp/chassis-shootout/broken + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.13" + + - name: Run intentionally broken candidate + id: broken + continue-on-error: true + run: | + python tools/run_chassis_ci_contract.py \ + --candidate broken-control \ + --candidate-revision fixture-broken-v1 \ + --candidate-command-json '["python","-c","print(\"no repair\")"]' \ + --verifier-command-json '["python","-c","from pathlib import Path; raise SystemExit(0 if Path(\"result.txt\").exists() else 1)"]' \ + --workdir "$WORKDIR" \ + --output "$EVIDENCE_DIR/result.json" + + - name: Assert negative control failed closed + if: always() + env: + BROKEN_OUTCOME: ${{ steps.broken.outcome }} + run: | + test "$BROKEN_OUTCOME" = "failure" + python - <<'PY' + import json + from pathlib import Path + r = json.loads(Path("logs/chassis-shootout/contract/broken/result.json").read_text()) + assert r["classification"] == "VERIFIED_FAIL", r + assert r["verifier_result"] == "FAIL", r + print(json.dumps({"negative_control": "PASS", "classification": r["classification"]})) + PY + + - name: Upload negative-control evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: chassis-contract-broken-${{ github.sha }} + retention-days: 30 + if-no-files-found: error + path: logs/chassis-shootout/contract/broken/ diff --git a/docs/decisions/0169-chassis-shootout-runner-contract.md b/docs/decisions/0169-chassis-shootout-runner-contract.md new file mode 100644 index 00000000..0784c6fd --- /dev/null +++ b/docs/decisions/0169-chassis-shootout-runner-contract.md @@ -0,0 +1,44 @@ +# 0169 — Chassis shootout runner contract + +## Status + +CANDIDATE — requires hosted GitHub Actions execution before acceptance. + +## Decision basis + +problem_class = comparative coding-agent runner reproducibility +decision = add one dependency-free, fail-closed runner contract for #99 +basis_type = PROJECT_INVARIANT + LOCAL_EVIDENCE +basis_ref = issues #84, #98, #99; existing provider-free GitHub Actions workflows +supported_claim = CodePro already separates command observation from verification and preserves raw hosted-CI evidence +applicability = Nano and Agentless require identical evidence semantics before any quality comparison +deviation = none; this does not execute either real upstream candidate yet + +## Contract + +The runner: + +- executes argv without shell interpretation; +- records candidate process outcome as observation only; +- runs an independent verifier; +- derives PASS/FAIL only from the verifier; +- separates timeout/launch errors as infrastructure failure; +- records runner identity and project/candidate revisions; +- always writes structured evidence before returning its final status. + +The hosted workflow proves the plumbing with deterministic fixtures labelled nano and agentless; those fixture labels are not claims that either upstream candidate executed. + +A deliberate broken control must: + +1. return a failing workflow step; +2. classify as VERIFIED_FAIL; +3. retain its JSON evidence artifact. + +## Boundaries + +CI_CONTRACT_VALIDATION != REAL_MODEL_BENCHMARK +FIXTURE_ADAPTER_PASS != UPSTREAM_CANDIDATE_PASS +COMMAND_EXIT != TASK_SUCCESS +IMPLEMENTED != EXECUTED + +Real Nano and Agentless baselines belong to #101 and #102 after #100 freezes the workload/verifier/budgets. diff --git a/tests/test_chassis_ci_contract.py b/tests/test_chassis_ci_contract.py new file mode 100644 index 00000000..17cba558 --- /dev/null +++ b/tests/test_chassis_ci_contract.py @@ -0,0 +1,80 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +RUNNER = ROOT / "tools" / "run_chassis_ci_contract.py" + + +class ChassisCiContractTests(unittest.TestCase): + def _run(self, candidate_code: str, verifier_code: str): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + output = td / "evidence" / "result.json" + cmd = [ + sys.executable, + str(RUNNER), + "--candidate", "fixture", + "--candidate-revision", "fixture-sha", + "--candidate-command-json", json.dumps([sys.executable, "-c", candidate_code]), + "--verifier-command-json", json.dumps([sys.executable, "-c", verifier_code]), + "--workdir", str(td / "work"), + "--output", str(output), + "--timeout-seconds", "5", + ] + cp = subprocess.run(cmd, text=True, capture_output=True, check=False) + self.assertTrue(output.exists(), cp.stderr) + return cp, json.loads(output.read_text(encoding="utf-8")) + + def test_verified_pass_is_owned_by_verifier_not_candidate_exit(self): + candidate = "from pathlib import Path; Path('ok.txt').write_text('ok')" + verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').read_text() == 'ok' else 1)" + cp, rec = self._run(candidate, verifier) + self.assertEqual(cp.returncode, 0) + self.assertEqual(rec["classification"], "VERIFIED_PASS") + self.assertEqual(rec["verifier_result"], "PASS") + self.assertFalse(rec["invariants"]["candidate_exit_is_task_success"]) + + def test_broken_candidate_fails_closed(self): + candidate = "print('no repair performed')" + verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').exists() else 1)" + cp, rec = self._run(candidate, verifier) + self.assertEqual(cp.returncode, 2) + self.assertEqual(rec["classification"], "VERIFIED_FAIL") + self.assertEqual(rec["verifier_result"], "FAIL") + + def test_nonzero_candidate_can_still_be_verified_if_workspace_is_correct(self): + candidate = "from pathlib import Path; Path('ok.txt').write_text('ok'); raise SystemExit(7)" + verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').read_text() == 'ok' else 1)" + cp, rec = self._run(candidate, verifier) + self.assertEqual(rec["candidate_observation"]["exit_code"], 7) + self.assertEqual(cp.returncode, 0) + self.assertEqual(rec["classification"], "VERIFIED_PASS") + + def test_missing_executable_is_infrastructure_failure(self): + with tempfile.TemporaryDirectory() as td: + td = Path(td) + output = td / "result.json" + cmd = [ + sys.executable, str(RUNNER), + "--candidate", "fixture", + "--candidate-revision", "fixture-sha", + "--candidate-command-json", json.dumps(["definitely-not-a-real-command-xyz"]), + "--verifier-command-json", json.dumps([sys.executable, "-c", "raise SystemExit(0)"]), + "--workdir", str(td / "work"), + "--output", str(output), + "--timeout-seconds", "5", + ] + cp = subprocess.run(cmd, text=True, capture_output=True, check=False) + rec = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(cp.returncode, 3) + self.assertEqual(rec["classification"], "INFRA_FAILURE") + self.assertEqual(rec["verifier_result"], "UNKNOWN") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/run_chassis_ci_contract.py b/tools/run_chassis_ci_contract.py new file mode 100644 index 00000000..53fede1d --- /dev/null +++ b/tools/run_chassis_ci_contract.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import argparse +import json +import os +import platform +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +def _run(argv: list[str], cwd: Path, timeout: float) -> dict[str, Any]: + started = time.time() + try: + cp = subprocess.run( + argv, + cwd=cwd, + text=True, + capture_output=True, + timeout=timeout, + check=False, + shell=False, + ) + return { + "argv": argv, + "exit_code": cp.returncode, + "stdout": cp.stdout, + "stderr": cp.stderr, + "timed_out": False, + "launch_error": None, + "wall_seconds": round(time.time() - started, 6), + } + except subprocess.TimeoutExpired as exc: + return { + "argv": argv, + "exit_code": None, + "stdout": exc.stdout or "", + "stderr": exc.stderr or "", + "timed_out": True, + "launch_error": None, + "wall_seconds": round(time.time() - started, 6), + } + except OSError as exc: + return { + "argv": argv, + "exit_code": None, + "stdout": "", + "stderr": "", + "timed_out": False, + "launch_error": f"{type(exc).__name__}: {exc}", + "wall_seconds": round(time.time() - started, 6), + } + + +def _runner_identity() -> dict[str, Any]: + return { + "platform": platform.platform(), + "machine": platform.machine(), + "python": sys.version, + "github_runner_os": os.getenv("RUNNER_OS"), + "github_runner_arch": os.getenv("RUNNER_ARCH"), + "github_image_os": os.getenv("ImageOS"), + "github_image_version": os.getenv("ImageVersion"), + "github_run_id": os.getenv("GITHUB_RUN_ID"), + "github_run_attempt": os.getenv("GITHUB_RUN_ATTEMPT"), + "github_sha": os.getenv("GITHUB_SHA"), + } + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--candidate", required=True) + p.add_argument("--candidate-revision", required=True) + p.add_argument("--candidate-command-json", required=True) + p.add_argument("--verifier-command-json", required=True) + p.add_argument("--workdir", required=True) + p.add_argument("--output", required=True) + p.add_argument("--timeout-seconds", type=float, default=60.0) + args = p.parse_args() + + candidate_argv = json.loads(args.candidate_command_json) + verifier_argv = json.loads(args.verifier_command_json) + if not isinstance(candidate_argv, list) or not all(isinstance(x, str) for x in candidate_argv): + raise SystemExit("candidate command must be a JSON string array") + if not isinstance(verifier_argv, list) or not all(isinstance(x, str) for x in verifier_argv): + raise SystemExit("verifier command must be a JSON string array") + + workdir = Path(args.workdir).resolve() + out = Path(args.output) + workdir.mkdir(parents=True, exist_ok=True) + out.parent.mkdir(parents=True, exist_ok=True) + + candidate = _run(candidate_argv, workdir, args.timeout_seconds) + verifier = _run(verifier_argv, workdir, args.timeout_seconds) + + infra_failure = any([ + candidate["launch_error"] is not None, + candidate["timed_out"], + verifier["launch_error"] is not None, + verifier["timed_out"], + ]) + + if infra_failure: + classification = "INFRA_FAILURE" + verifier_result = "UNKNOWN" + rc = 3 + elif verifier["exit_code"] == 0: + classification = "VERIFIED_PASS" + verifier_result = "PASS" + rc = 0 + else: + classification = "VERIFIED_FAIL" + verifier_result = "FAIL" + rc = 2 + + record = { + "schema_version": 1, + "candidate": args.candidate, + "candidate_revision": args.candidate_revision, + "project_commit": os.getenv("GITHUB_SHA"), + "runner_identity": _runner_identity(), + "timeout_seconds": args.timeout_seconds, + "candidate_observation": candidate, + "verifier_observation": verifier, + "verifier_result": verifier_result, + "classification": classification, + "invariants": { + "shell_interpretation": False, + "candidate_exit_is_task_success": False, + "verifier_is_independent": True, + "silent_fallback": False, + }, + } + out.write_text(json.dumps(record, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps({ + "candidate": args.candidate, + "classification": classification, + "verifier_result": verifier_result, + "output": str(out), + }, sort_keys=True)) + return rc + + +if __name__ == "__main__": + raise SystemExit(main())