Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/verification.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,11 @@ top-level `tests.evidence` list. This is separate from the unchanged code-host
}
```

Every entry in a check's `evidence` and `attempts` arrays must be a JSON
object. A non-object entry (including a string) is malformed; the runtime
rejects the report and caps the conclusion at `not_verified` rather than
coercing text into evidence.

Freeze each check before execution. A result names the same check ID, command,
executor/profile and context. The runtime supplies frozen check identities to
later code stages. Copy that identity, including its owning executor/profile,
Expand Down
8 changes: 8 additions & 0 deletions scripts/verification_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,14 @@ def normalize_tests(tests: Mapping[str, Any], head_sha: str, *,
prior = CheckResult.from_dict({**previous, "id": check.id})
if prior not in plan.results:
plan = plan.record(prior)
for field in ("evidence", "attempts"):
values = entry.get(field, ())
if not isinstance(values, (list, tuple)):
raise VerificationError(f"check {check.id} {field} must be an array")
for index, item in enumerate(values):
if not isinstance(item, Mapping):
raise VerificationError(
f"check {check.id} {field}[{index}] must be an object, got {type(item).__name__}")
result = CheckResult(check.id, entry["head_sha"], entry["result"],
tuple(entry.get("evidence", ())), entry.get("cause", ""),
tuple(entry.get("attempts", ())))
Expand Down
3 changes: 3 additions & 0 deletions skills/cc-verify/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ obligations.
Each result also carries `head_sha`, `result: pass|fail|unverified`,
`evidence` (integer `exit_code`, `executed` count or null), `cause` and
`attempts`. The enclosing stage result names its current `head_sha` too.
Each `evidence` and `attempts` array entry must be a JSON object. Do not send
text entries or rely on the runtime to convert them; malformed entries make
the report invalid and cap its conclusion at `not_verified`.
An unexecuted check is explicitly `unverified` with a cause and the recovery
attempts actually made; do not invent an exit code. Keep the check's identity
when rerunning it and retain previous results. Never replace a local
Expand Down
27 changes: 27 additions & 0 deletions tests/test_verification_plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,33 @@ def test_invalid_followup_never_raises_failed_agent_or_frozen_conclusion(self):
"version": 99, "conclusion": conclusion, "passed": True}}, plan)
self.assertEqual(rc._legacy_tests(payload)["verification"], "failed")

def test_v2_rejects_string_evidence_and_attempts_with_check_name(self):
plan = vp.VerificationPlan(criteria()).extend((check(),))
for field in ("evidence", "attempts"):
entry = report(**{field: ["text"]})
payload = {"version": 2, "conclusion": "verified", "checks": [entry]}
with self.subTest(field=field), self.assertRaisesRegex(
vp.VerificationError, rf"check check-1 {field}\[0\].*str"):
vp.normalize_tests(payload, HEAD, plan=plan)

def test_v1_string_evidence_rejects_with_plan_and_stays_unchanged_without_one(self):
legacy = {"version": 1, "conclusion": "verified", "evidence": ["text"]}
plan = vp.VerificationPlan(criteria()).extend((check(),))
with self.assertRaisesRegex(vp.VerificationError, "invalid legacy evidence"):
vp.normalize_tests(legacy, HEAD, plan=plan)
unchanged, updated = vp.normalize_tests(legacy, HEAD)
self.assertEqual(unchanged, legacy)
self.assertIsNone(updated)

def test_string_check_entries_reach_cycle_as_not_verified(self):
for field in ("evidence", "attempts"):
payload, _ = rc._normalize_test_payload({"head_sha": HEAD, "tests": {
"version": 2, "conclusion": "verified", "passed": True,
"checks": [report(**{field: ["text"]})]}})
with self.subTest(field=field):
self.assertEqual(payload["tests"]["conclusion"], "not_verified")
self.assertFalse(payload["tests"]["passed"])

def test_legacy_unmatched_failures_are_retained(self):
plan = vp.VerificationPlan(criteria()).extend((check(),))
payload, updated = rc._normalize_test_payload({"status": "IMPLEMENTED", "head_sha": HEAD,
Expand Down
Loading