Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
bump: minor
---

### Added

- `run-e2e` has a `findings.json` schema for one live verification pass: each finding's id, feature, part, kind, where, entry points, severity, `stated` or `unstated` label and evidence paths, and the pass's build plus each entry's outcome, wall time and tokens.
- `run-e2e`'s `scripts/findings-compare.sh` compares two `findings.json` files with no model call. It labels each finding id NEW, PRE-EXISTING, GONE or not checked. An id only shows as GONE when the newer pass drove every entry point the earlier finding listed, and every finding is NEW when no baseline is given.
139 changes: 139 additions & 0 deletions plugins/testing/skills/run-e2e/schemas/findings.schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:claude-code-plugin:testing:run-e2e-findings",
"title": "Findings",
"description": "One live verification pass over a feature map: the pass's build, each driven or skipped entry, and one record per finding id. scripts/findings-compare.sh reads two of these files and labels each finding id NEW, PRE-EXISTING, GONE or not checked.",
"type": "object",
"additionalProperties": false,
"required": ["pass", "findings"],
"properties": {
"pass": {
"type": "object",
"additionalProperties": false,
"required": ["build", "entries"],
"properties": {
"build": {
"type": "string",
"minLength": 1,
"description": "The build the pass drove, such as a commit SHA or a release tag. A comparison report shows both passes' builds in its header."
},
"entries": {
"type": "array",
"items": { "$ref": "#/$defs/Entry" },
"description": "One row per entry the pass listed, driven or not. A finding only shows as GONE when every entry point it listed in the earlier pass was driven here."
}
}
},
"findings": {
"type": "array",
"items": { "$ref": "#/$defs/Finding" },
"description": "One record per finding id. Two records with the same id are invalid, so a failure seen through several entry points is a single record that lists each one."
}
},
"$defs": {
"Part": {
"type": ["string", "null"],
"minLength": 1,
"description": "The feature's part (the <part> of <feature>/<part>), or null when the pass drove whole features."
},
"Entry": {
"type": "object",
"additionalProperties": false,
"required": ["feature", "part", "entry_point", "outcome", "reason", "wall_time_seconds", "tokens", "unmeasured_reason"],
"properties": {
"feature": { "type": "string", "minLength": 1, "description": "The feature id, its index path without features/ and the trailing /README.md." },
"part": { "$ref": "#/$defs/Part" },
"entry_point": { "type": "string", "minLength": 1, "description": "The entry point's text, as the feature file lists it." },
"outcome": {
"type": "string",
"enum": ["passed", "failed", "not checked"],
"description": "passed or failed when the entry was driven. not checked when it was blocked, not driven or left out by a cap; such an entry is never a finding."
},
"reason": {
"type": ["string", "null"],
"description": "Why the entry was not checked, such as cap or the blocked: line. Null when it was driven."
},
"wall_time_seconds": {
"type": ["number", "null"],
"minimum": 0,
"description": "Measured wall time for the entry. Null when not measured, never an estimate."
},
"tokens": {
"type": ["integer", "null"],
"minimum": 0,
"description": "Measured tokens the entry spent. Null when not measured, never an estimate."
},
"unmeasured_reason": {
"type": ["string", "null"],
"description": "Why wall_time_seconds or tokens is null. Null when both were measured."
}
},
"allOf": [
{
"if": { "properties": { "outcome": { "const": "not checked" } } },
"then": { "properties": { "reason": { "type": "string", "minLength": 1 } } },
"else": { "properties": { "reason": { "type": "null" } } }
},
{
"if": {
"anyOf": [
{ "properties": { "wall_time_seconds": { "type": "null" } } },
{ "properties": { "tokens": { "type": "null" } } }
]
},
"then": { "properties": { "unmeasured_reason": { "type": "string", "minLength": 1 } } },
"else": { "properties": { "unmeasured_reason": { "type": "null" } } }
}
]
},
"Finding": {
"type": "object",
"additionalProperties": false,
"required": ["id", "feature", "part", "kind", "where", "entry_points", "severity", "label", "evidence"],
"properties": {
"id": {
"type": "string",
"pattern": "^[0-9a-f]{8,64}$",
"description": "A short hex hash of feature, part, kind and where, without the entry point, so it stays the same across passes. Passes are matched by exact id."
},
"feature": { "type": "string", "minLength": 1 },
"part": { "$ref": "#/$defs/Part" },
"kind": {
"type": "string",
"enum": ["result-not-met", "error", "crash", "refusal", "no-output", "garbled-output"],
"description": "result-not-met for a stated result or proof end state the run did not reach. The other five are the closed list of breakage kinds for a defect the map states nothing about."
},
"where": {
"type": "string",
"pattern": "^([1-9][0-9]*|proof|run)$",
"description": "The drive step number, proof for the proof step, or run for the run as a whole."
},
"entry_points": {
"type": "array",
"minItems": 1,
"uniqueItems": true,
"items": { "type": "string", "minLength": 1 },
"description": "Every entry point the finding showed at."
},
"severity": { "type": "string", "enum": ["critical", "high", "medium", "low"] },
"label": {
"type": "string",
"enum": ["stated", "unstated"],
"description": "stated when the finding fails a result the feature file states, unstated for breakage the map states nothing about."
},
"evidence": {
"type": "array",
"items": { "type": "string", "minLength": 1 },
"description": "Evidence paths, relative to the pass's run directory."
}
},
"allOf": [
{
"if": { "properties": { "label": { "const": "stated" } } },
"then": { "properties": { "kind": { "const": "result-not-met" } } },
"else": { "properties": { "kind": { "not": { "const": "result-not-met" } } } }
}
]
}
}
}
131 changes: 131 additions & 0 deletions plugins/testing/skills/run-e2e/scripts/findings-compare.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
#!/usr/bin/env bash
# Compares two live verification passes, each a findings.json
# (../schemas/findings.schema.json), and labels every finding id. No model
# call: ids are matched exactly.
#
# findings-compare.sh [--baseline <findings.json>] <findings.json>
#
# The positional file is the newer pass. Labels:
# NEW the id is in the newer pass and not in the baseline
# PRE-EXISTING the id is in both
# GONE the id is only in the baseline, and the newer pass drove
# (passed or failed) every entry point the baseline finding
# lists, for its feature and part
# not checked the id is only in the baseline, and at least one of its entry
# points was not driven in the newer pass; not_driven names
# each one and why
# With no --baseline every finding in the newer pass is NEW.
#
# Prints one JSON object on stdout: builds (baseline, newer), counts per
# label, and findings, each the finding's record plus `compare` (its label),
# ordered NEW, PRE-EXISTING, GONE, not checked, then by severity (critical
# first), then by id. A PRE-EXISTING record is the newer pass's; a GONE or
# not checked record is the baseline's.
#
# Exit 0 on a comparison, 2 on a usage error, a missing jq, or a file that is
# not a findings.json (unreadable, not JSON, a missing field, a duplicate id).
set -uo pipefail

usage() {
echo "usage: findings-compare.sh [--baseline <findings.json>] <findings.json>" >&2
exit 2
}

baseline=""
newer=""
while (($#)); do
case "$1" in
--baseline)
(($# >= 2)) || usage
baseline="$2"
shift 2
;;
-*) usage ;;
*)
[[ -z "$newer" ]] || usage
newer="$1"
shift
;;
esac
done
[[ -n "$newer" ]] || usage

command -v jq >/dev/null 2>&1 || {
echo "findings-compare.sh: jq is required" >&2
exit 2
}

# check_file <path>: exit 2 unless the file holds the fields this script reads.
check_file() {
[[ -f "$1" && -r "$1" ]] || {
echo "findings-compare.sh: cannot read $1" >&2
exit 2
}
local problem
if ! problem="$(jq -rs '
def str: type == "string" and length > 0;
def part_ok: .part == null or (.part | str);
if length != 1 then "not exactly one JSON value" else .[0] |
if type != "object" then "not a JSON object"
elif (.pass | type) != "object" then "no pass object"
elif (.pass.build | str) | not then "pass.build is not a non-empty string"
elif (.pass.entries | type) != "array" then "pass.entries is not an array"
elif any(.pass.entries[]; (type != "object") or ((.feature | str) | not) or (part_ok | not)
or ((.entry_point | str) | not) or ((.outcome | IN("passed", "failed", "not checked")) | not))
then "an entry lacks feature, part, entry_point or a known outcome"
elif (.findings | type) != "array" then "findings is not an array"
elif any(.findings[]; (type != "object") or ((.id | str) | not) or ((.feature | str) | not)
or (part_ok | not) or ((.entry_points | type) != "array")
or ((.entry_points | length) == 0) or any(.entry_points[]; str | not))
then "a finding lacks id, feature, part or entry_points"
elif ([.findings[].id] | length) != ([.findings[].id] | unique | length)
then "two findings share an id"
else empty end end' "$1" 2>/dev/null)"; then
echo "findings-compare.sh: $1 is not JSON" >&2
exit 2
fi
[[ -z "$problem" ]] || {
echo "findings-compare.sh: $1: $problem" >&2
exit 2
}
}

check_file "$newer"
if [[ -n "$baseline" ]]; then
check_file "$baseline"
base_args=(--slurpfile base "$baseline")
else
base_args=(--argjson base '[]')
fi

# shellcheck disable=SC2016 # $vars are jq's
jq -n --slurpfile cur "$newer" "${base_args[@]}" '
def labels: ["NEW", "PRE-EXISTING", "GONE", "not checked"];
def severities: ["critical", "high", "medium", "low"];
def key($f; $p; $e): [$f, $p, $e] | tojson;
($cur[0]) as $n
| ($base[0] // null) as $b
| ([$n.findings[].id]) as $nids
| ([($b // {findings: []}).findings[].id]) as $bids
| ([$n.pass.entries[] | select(.outcome != "not checked")
| key(.feature; .part; .entry_point)]) as $driven
| ([$n.pass.entries[] | select(.outcome == "not checked")
| {key: key(.feature; .part; .entry_point), value: (.reason // "not checked")}]
| from_entries) as $skipped
| ([$n.findings[]
| . + {compare: (if (.id | IN($bids[])) then "PRE-EXISTING" else "NEW" end)}]
+ [($b // {findings: []}).findings[] | select((.id | IN($nids[])) | not)
| . as $f
| [$f.entry_points[] | key($f.feature; $f.part; .) as $k
| select(($k | IN($driven[])) | not)
| {entry_point: ., reason: ($skipped[$k] // "not in the newer pass")}] as $missing
| if ($missing | length) == 0 then $f + {compare: "GONE"}
else $f + {compare: "not checked", not_driven: $missing} end])
| sort_by([(.compare as $c | labels | index($c)),
(.severity as $s | severities | index($s) // 99), .id]) as $all
| {
builds: {baseline: ($b.pass.build // null), newer: $n.pass.build},
counts: (labels | map({key: ., value: (. as $l | [$all[] | select(.compare == $l)] | length)})
| from_entries),
findings: $all
}'
Loading
Loading