test: add chassis shootout runner contract

.github/workflows/chassis-shootout-contract.yml

@@ -0,0 +1,130 @@
+name: Chassis shootout runner contract
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - ".github/workflows/chassis-shootout-contract.yml"
+      - "tools/run_chassis_ci_contract.py"
+      - "tests/test_chassis_ci_contract.py"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: chassis-shootout-contract-${{ github.ref }}-${{ github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  contract-tests:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.13"
+
+      - name: Run runner contract tests
+        run: python -m unittest -v tests.test_chassis_ci_contract
+
+  deterministic-adapter-smoke:
+    needs: contract-tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        candidate: [nano, agentless]
+    env:
+      EVIDENCE_DIR: logs/chassis-shootout/contract/${{ matrix.candidate }}
+      WORKDIR: .tmp/chassis-shootout/${{ matrix.candidate }}
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.13"
+
+      - name: Record runner identity
+        run: |
+          mkdir -p "$EVIDENCE_DIR"
+          git rev-parse HEAD | tee "$EVIDENCE_DIR/project-commit.txt"
+          python -VV | tee "$EVIDENCE_DIR/python.txt"
+          uname -a | tee "$EVIDENCE_DIR/uname.txt"
+
+      - name: Run deterministic candidate fixture
+        run: |
+          python tools/run_chassis_ci_contract.py \
+            --candidate "${{ matrix.candidate }}" \
+            --candidate-revision "fixture-v1" \
+            --candidate-command-json '["python","-c","from pathlib import Path; Path(\"result.txt\").write_text(\"fixed\\n\")"]' \
+            --verifier-command-json '["python","-c","from pathlib import Path; raise SystemExit(0 if Path(\"result.txt\").read_text() == \"fixed\\n\" else 1)"]' \
+            --workdir "$WORKDIR" \
+            --output "$EVIDENCE_DIR/result.json"
+
+      - name: Upload candidate evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: chassis-contract-${{ matrix.candidate }}-${{ github.sha }}
+          retention-days: 30
+          if-no-files-found: error
+          path: logs/chassis-shootout/contract/${{ matrix.candidate }}/
+
+  negative-control:
+    needs: contract-tests
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    env:
+      EVIDENCE_DIR: logs/chassis-shootout/contract/broken
+      WORKDIR: .tmp/chassis-shootout/broken
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.13"
+
+      - name: Run intentionally broken candidate
+        id: broken
+        continue-on-error: true
+        run: |
+          python tools/run_chassis_ci_contract.py \
+            --candidate broken-control \
+            --candidate-revision fixture-broken-v1 \
+            --candidate-command-json '["python","-c","print(\"no repair\")"]' \
+            --verifier-command-json '["python","-c","from pathlib import Path; raise SystemExit(0 if Path(\"result.txt\").exists() else 1)"]' \
+            --workdir "$WORKDIR" \
+            --output "$EVIDENCE_DIR/result.json"
+
+      - name: Assert negative control failed closed
+        if: always()
+        env:
+          BROKEN_OUTCOME: ${{ steps.broken.outcome }}
+        run: |
+          test "$BROKEN_OUTCOME" = "failure"
+          python - <<'PY'
+          import json
+          from pathlib import Path
+          r = json.loads(Path("logs/chassis-shootout/contract/broken/result.json").read_text())
+          assert r["classification"] == "VERIFIED_FAIL", r
+          assert r["verifier_result"] == "FAIL", r
+          print(json.dumps({"negative_control": "PASS", "classification": r["classification"]}))
+          PY
+
+      - name: Upload negative-control evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: chassis-contract-broken-${{ github.sha }}
+          retention-days: 30
+          if-no-files-found: error
+          path: logs/chassis-shootout/contract/broken/

docs/decisions/0169-chassis-shootout-runner-contract.md

@@ -0,0 +1,44 @@
+# 0169 — Chassis shootout runner contract
+
+## Status
+
+CANDIDATE — requires hosted GitHub Actions execution before acceptance.
+
+## Decision basis
+
+problem_class = comparative coding-agent runner reproducibility  
+decision = add one dependency-free, fail-closed runner contract for #99  
+basis_type = PROJECT_INVARIANT + LOCAL_EVIDENCE  
+basis_ref = issues #84, #98, #99; existing provider-free GitHub Actions workflows  
+supported_claim = CodePro already separates command observation from verification and preserves raw hosted-CI evidence  
+applicability = Nano and Agentless require identical evidence semantics before any quality comparison  
+deviation = none; this does not execute either real upstream candidate yet
+
+## Contract
+
+The runner:
+
+- executes argv without shell interpretation;
+- records candidate process outcome as observation only;
+- runs an independent verifier;
+- derives PASS/FAIL only from the verifier;
+- separates timeout/launch errors as infrastructure failure;
+- records runner identity and project/candidate revisions;
+- always writes structured evidence before returning its final status.
+
+The hosted workflow proves the plumbing with deterministic fixtures labelled nano and agentless; those fixture labels are not claims that either upstream candidate executed.
+
+A deliberate broken control must:
+
+1. return a failing workflow step;
+2. classify as VERIFIED_FAIL;
+3. retain its JSON evidence artifact.
+
+## Boundaries
+
+CI_CONTRACT_VALIDATION != REAL_MODEL_BENCHMARK  
+FIXTURE_ADAPTER_PASS != UPSTREAM_CANDIDATE_PASS  
+COMMAND_EXIT != TASK_SUCCESS  
+IMPLEMENTED != EXECUTED
+
+Real Nano and Agentless baselines belong to #101 and #102 after #100 freezes the workload/verifier/budgets.

tests/test_chassis_ci_contract.py

@@ -0,0 +1,80 @@
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+RUNNER = ROOT / "tools" / "run_chassis_ci_contract.py"
+
+
+class ChassisCiContractTests(unittest.TestCase):
+    def _run(self, candidate_code: str, verifier_code: str):
+        with tempfile.TemporaryDirectory() as td:
+            td = Path(td)
+            output = td / "evidence" / "result.json"
+            cmd = [
+                sys.executable,
+                str(RUNNER),
+                "--candidate", "fixture",
+                "--candidate-revision", "fixture-sha",
+                "--candidate-command-json", json.dumps([sys.executable, "-c", candidate_code]),
+                "--verifier-command-json", json.dumps([sys.executable, "-c", verifier_code]),
+                "--workdir", str(td / "work"),
+                "--output", str(output),
+                "--timeout-seconds", "5",
+            ]
+            cp = subprocess.run(cmd, text=True, capture_output=True, check=False)
+            self.assertTrue(output.exists(), cp.stderr)
+            return cp, json.loads(output.read_text(encoding="utf-8"))
+
+    def test_verified_pass_is_owned_by_verifier_not_candidate_exit(self):
+        candidate = "from pathlib import Path; Path('ok.txt').write_text('ok')"
+        verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').read_text() == 'ok' else 1)"
+        cp, rec = self._run(candidate, verifier)
+        self.assertEqual(cp.returncode, 0)
+        self.assertEqual(rec["classification"], "VERIFIED_PASS")
+        self.assertEqual(rec["verifier_result"], "PASS")
+        self.assertFalse(rec["invariants"]["candidate_exit_is_task_success"])
+
+    def test_broken_candidate_fails_closed(self):
+        candidate = "print('no repair performed')"
+        verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').exists() else 1)"
+        cp, rec = self._run(candidate, verifier)
+        self.assertEqual(cp.returncode, 2)
+        self.assertEqual(rec["classification"], "VERIFIED_FAIL")
+        self.assertEqual(rec["verifier_result"], "FAIL")
+
+    def test_nonzero_candidate_can_still_be_verified_if_workspace_is_correct(self):
+        candidate = "from pathlib import Path; Path('ok.txt').write_text('ok'); raise SystemExit(7)"
+        verifier = "from pathlib import Path; raise SystemExit(0 if Path('ok.txt').read_text() == 'ok' else 1)"
+        cp, rec = self._run(candidate, verifier)
+        self.assertEqual(rec["candidate_observation"]["exit_code"], 7)
+        self.assertEqual(cp.returncode, 0)
+        self.assertEqual(rec["classification"], "VERIFIED_PASS")
+
+    def test_missing_executable_is_infrastructure_failure(self):
+        with tempfile.TemporaryDirectory() as td:
+            td = Path(td)
+            output = td / "result.json"
+            cmd = [
+                sys.executable, str(RUNNER),
+                "--candidate", "fixture",
+                "--candidate-revision", "fixture-sha",
+                "--candidate-command-json", json.dumps(["definitely-not-a-real-command-xyz"]),
+                "--verifier-command-json", json.dumps([sys.executable, "-c", "raise SystemExit(0)"]),
+                "--workdir", str(td / "work"),
+                "--output", str(output),
+                "--timeout-seconds", "5",
+            ]
+            cp = subprocess.run(cmd, text=True, capture_output=True, check=False)
+            rec = json.loads(output.read_text(encoding="utf-8"))
+        self.assertEqual(cp.returncode, 3)
+        self.assertEqual(rec["classification"], "INFRA_FAILURE")
+        self.assertEqual(rec["verifier_result"], "UNKNOWN")
+
+
+if __name__ == "__main__":
+    unittest.main()