diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index beb3e131..779734d5 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -104,6 +104,32 @@ jobs: body: | Refreshes the quality-gate baseline after a merge to master. + # The gate reads the baseline it is given, so a PR that loosens a + # .gate-baseline.json would pass the gate it just weakened. + baseline-tightened: + name: Baseline only tightens + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Compare each changed baseline with the PR's base + run: | + verdict=0 + while IFS= read -r file; do + before="$RUNNER_TEMP/before.json" + git show "$BASE:$file" > "$before" 2>/dev/null || rm -f "$before" + echo "== $file" + ./scripts/quality-gate/baseline-tightened.sh "$before" "$file" || verdict=1 + done < <(git diff --name-only "$BASE"...HEAD -- 'packages/*/.gate-baseline.json') + exit "$verdict" + env: + BASE: ${{ github.event.pull_request.base.sha }} + # Validates the in-code capability annotations against the canonical matrix spec: # catches a mistyped feature key or a feature reported with conflicting statuses. capability-matrix: diff --git a/scripts/quality-gate/README.md b/scripts/quality-gate/README.md index 39ddaaa4..8169c6e0 100644 --- a/scripts/quality-gate/README.md +++ b/scripts/quality-gate/README.md @@ -143,6 +143,16 @@ FIXTURE_KEEP=1 ./scripts/quality-gate/fixtures/run-fixtures.sh # keep work dir Change the gate's behavior → add or update the fixture that pins it. A behavior with no fixture is untested. +## Baselines only tighten + +The gate reads the baseline it is given, so a PR that loosens one would pass the gate it +just weakened. `baseline-tightened.sh` runs on every PR against each changed +`.gate-baseline.json` and fails when warnings rose, coverage fell, an identity field +changed or the file was deleted. The gate itself only ever writes a stricter baseline, so +a looser one can only come from a hand edit. + +```sh +./scripts/quality-gate/fixtures-baseline-tightened/run.sh # its cases ## Which CI jobs a change needs `changed-areas.sh` classifies a diff, and each workflow skips the jobs it cannot affect: @@ -170,7 +180,9 @@ lib/warnings.sh analyzer warnings + per-package baseline I/O lib/coverage.sh line coverage + per-package baseline I/O lib/stages.sh one definition of each stage, used for both scopes format-suggest.sh prints the format diff for a changed file (dev helper) -changed-areas.sh classifies a diff into the areas CI jobs care about +baseline-tightened.sh fails when a baseline got looser fixtures/ the gate's own test suite +fixtures-baseline-tightened/ cases for baseline-tightened.sh fixtures-changed-areas/ cases for changed-areas.sh +changed-areas.sh classifies a diff into the areas CI jobs care about ``` diff --git a/scripts/quality-gate/baseline-tightened.sh b/scripts/quality-gate/baseline-tightened.sh new file mode 100755 index 00000000..11763989 --- /dev/null +++ b/scripts/quality-gate/baseline-tightened.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# +# baseline-tightened.sh — exit 0 when the per-package +# .gate-baseline.json only got stricter, 1 when any measurement got looser. +# +# The gate reads the baseline it is given, so a PR that loosens one passes the +# gate it just weakened. This check closes that: the gate itself only ever moves +# a baseline toward stricter (warnings down, coverage up), so a looser value can +# only come from a hand edit. +# +# before missing a new package's first baseline: allowed. +# after missing a deleted baseline: the next run would re-establish it +# from whatever it measures, so this is a loosening. +# package/project/... identity fields must not change. +# warnings.total must not rise. +# warnings.byCode. must not rise; a code absent before counted as 0. +# coverage.hermeticLine must not fall, and must not disappear. +# updated free. +# +set -euo pipefail + +[[ $# -eq 2 ]] || { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 3; } +BEFORE="$1"; AFTER="$2" + +command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 3; } + +[[ -f "$BEFORE" ]] || { echo "new baseline: nothing to compare"; exit 0; } +[[ -f "$AFTER" ]] || { echo "LOOSENED: baseline deleted"; exit 1; } + +loosenings="$(jq -rn --slurpfile before "$BEFORE" --slurpfile after "$AFTER" ' + ($before[0]) as $b | ($after[0]) as $a + | [ + ( ["schema", "package", "project", "testProject"][] + | select($b[.] != null and $a[.] != $b[.]) + | "\(.) changed: \($b[.]) -> \($a[.])" ), + ( select($b.warnings.total != null + and (($a.warnings.total // infinite) > $b.warnings.total)) + | "warnings.total rose: \($b.warnings.total) -> \($a.warnings.total)" ), + ( ($a.warnings.byCode // {}) | to_entries[] + | select(.value > ($b.warnings.byCode[.key] // 0)) + | "warnings.byCode.\(.key) rose: \($b.warnings.byCode[.key] // 0) -> \(.value)" ), + ( select($b.coverage.hermeticLine != null + and (($a.coverage.hermeticLine // -1) < $b.coverage.hermeticLine)) + | "coverage.hermeticLine fell: \($b.coverage.hermeticLine) -> \($a.coverage.hermeticLine)" ) + ] + | .[] +')" + +if [[ -n "$loosenings" ]]; then + while IFS= read -r line; do echo "LOOSENED: $line"; done <<<"$loosenings" + exit 1 +fi +echo "tightened or unchanged" diff --git a/scripts/quality-gate/fixtures-baseline-tightened/run.sh b/scripts/quality-gate/fixtures-baseline-tightened/run.sh new file mode 100755 index 00000000..88cec16d --- /dev/null +++ b/scripts/quality-gate/fixtures-baseline-tightened/run.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# run.sh — the test for baseline-tightened.sh. +# +# Each case is inline: a before/after baseline pair and the verdict it must +# produce. A wrong answer here lets a PR loosen the gate unnoticed, or blocks a +# legitimate baseline refresh, so every rule in the script's header has a case. +# A "loosened" verdict must come with a LOOSENED line, so a jq error can never +# pass for a detected loosening. +# +# run.sh run every case +# +# Exit 0 if every case passed, 1 otherwise. + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TIGHTENED="$HERE/../baseline-tightened.sh" +RUNROOT="${FIXTURE_RUNROOT:-${TMPDIR:-/tmp}/baseline-tightened-fixtures.$$}" + +command -v jq >/dev/null || { echo "jq required" >&2; exit 3; } + +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + RED=$'\033[31m'; GRN=$'\033[32m'; DIM=$'\033[2m'; BLD=$'\033[1m'; OFF=$'\033[0m' +else RED=""; GRN=""; DIM=""; BLD=""; OFF=""; fi + +mkdir -p "$RUNROOT" +trap '[[ -n "${FIXTURE_KEEP:-}" ]] || rm -rf "$RUNROOT"' EXIT + +PASS=0; FAIL=0 + +baseline_case() { + local name="$1" expected="$2" before="$3" after="$4" + local dir="$RUNROOT/${name// /-}" + mkdir -p "$dir" + [[ -n "$before" ]] && printf '%s' "$before" >"$dir/before.json" + [[ -n "$after" ]] && printf '%s' "$after" >"$dir/after.json" + + bash "$TIGHTENED" "$dir/before.json" "$dir/after.json" >"$dir/out.log" 2>&1 + local rc=$? + local verdict="error (exit $rc): $(tr '\n' ' ' <"$dir/out.log")" + [[ $rc -eq 0 ]] && verdict=tightened + [[ $rc -eq 1 ]] && grep -q '^LOOSENED: ' "$dir/out.log" && verdict=loosened + + if [[ "$expected" == "$verdict" ]]; then + printf " ${GRN}PASS${OFF} %s\n" "$name"; PASS=$((PASS+1)) + else + printf " ${RED}FAIL${OFF} %s\n ${DIM}expected: %s${OFF}\n ${DIM}actual: %s${OFF}\n" \ + "$name" "$expected" "$verdict" + FAIL=$((FAIL+1)) + fi +} + +BASE='{"schema":1,"package":"Gotrue","project":"Gotrue/Gotrue.csproj","testProject":"Gotrue.Tests/Gotrue.Tests.csproj","warnings":{"total":10,"byCode":{"CS1591":6,"CS8618":4}},"updated":"2026-10-01T00:00:00Z","coverage":{"hermeticLine":60.5}}' +with() { jq -c "$1" <<<"$BASE"; } + +baseline_case "unchanged" tightened "$BASE" "$BASE" +baseline_case "only updated moved" tightened "$BASE" "$(with '.updated = "2026-10-30T00:00:00Z"')" +baseline_case "warnings lowered" tightened "$BASE" "$(with '.warnings = {"total":8,"byCode":{"CS1591":4,"CS8618":4}}')" +baseline_case "code dropped to zero" tightened "$BASE" "$(with '.warnings = {"total":6,"byCode":{"CS1591":6}}')" +baseline_case "coverage raised" tightened "$BASE" "$(with '.coverage.hermeticLine = 62.1')" +baseline_case "coverage added" tightened "$(with 'del(.coverage)')" "$BASE" +baseline_case "new package" tightened "" "$BASE" +baseline_case "warnings total rose" loosened "$BASE" "$(with '.warnings.total = 11')" +baseline_case "warnings total removed" loosened "$BASE" "$(with 'del(.warnings.total)')" +baseline_case "one code rose" loosened "$BASE" "$(with '.warnings.byCode.CS8618 = 5')" +baseline_case "new code appeared" loosened "$BASE" "$(with '.warnings.byCode.CS0168 = 1')" +baseline_case "coverage fell" loosened "$BASE" "$(with '.coverage.hermeticLine = 60.4')" +baseline_case "coverage removed" loosened "$BASE" "$(with 'del(.coverage)')" +baseline_case "project changed" loosened "$BASE" "$(with '.project = "Other/Other.csproj"')" +baseline_case "baseline deleted" loosened "$BASE" "" + +echo +if [[ $FAIL -eq 0 ]]; then + echo "${GRN}${BLD}cases: $PASS/$((PASS+FAIL)) passed${OFF}"; exit 0 +else + echo "${RED}${BLD}cases: $FAIL/$((PASS+FAIL)) failed${OFF}"; exit 1 +fi diff --git a/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/expect.json b/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/expect.json new file mode 100644 index 00000000..0b849bb7 --- /dev/null +++ b/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/expect.json @@ -0,0 +1,11 @@ +{ + "description": "baseline-overwrite with a coverage baseline already recorded: lowering warnings must rewrite only .warnings and .updated. Guards the ratchet slide where rewriting the whole file dropped coverage.hermeticLine and the next stage re-established it lower.", + "mode": "--fast --overwrite-baseline", + "exit": 0, + "verdict": "PARTIAL", + "baselineWrite": true, + "baselineKeeps": [".coverage.hermeticLine"], + "stages": [ + { "id": "1a", "status": "PASS" } + ] +} diff --git a/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/overlay/.gate-baseline.json b/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/overlay/.gate-baseline.json new file mode 100644 index 00000000..874c8e9a --- /dev/null +++ b/scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/overlay/.gate-baseline.json @@ -0,0 +1,9 @@ +{ + "schema": 1, + "package": "baseline-overwrite-keeps-coverage", + "project": "Sample/Sample.csproj", + "testProject": "Sample.Tests/Sample.Tests.csproj", + "warnings": { "total": 3, "byCode": {} }, + "updated": "2026-08-12T00:00:00Z", + "coverage": { "hermeticLine": 99.5 } +} diff --git a/scripts/quality-gate/fixtures/run-fixtures.sh b/scripts/quality-gate/fixtures/run-fixtures.sh index c596d785..df9144bc 100755 --- a/scripts/quality-gate/fixtures/run-fixtures.sh +++ b/scripts/quality-gate/fixtures/run-fixtures.sh @@ -155,6 +155,14 @@ for fx in "${fixtures[@]}"; do fi fi + # Optional: keys a baseline write must leave as they were (expect.baselineKeeps: [jq paths]). + while IFS= read -r path; do + [[ -z "$path" ]] && continue + was="$(jq -c "$path" <<<"$bl_before")" + now="$(jq -c "$path" "$bl" 2>/dev/null || echo __ABSENT__)" + assert "baseline $path" "$was" "$now" + done < <(jq -r '.baselineKeeps // [] | .[]' "$exp") + if [[ -z "$miss" ]]; then printf " ${GRN}PASS${OFF} %-18s ${DIM}%s${OFF}\n" "$name" "$(jq -r '.description // ""' "$exp")" PASS=$((PASS+1)) diff --git a/scripts/quality-gate/lib/coverage.sh b/scripts/quality-gate/lib/coverage.sh index 4b57d5d9..4b0a5a96 100644 --- a/scripts/quality-gate/lib/coverage.sh +++ b/scripts/quality-gate/lib/coverage.sh @@ -83,12 +83,9 @@ coverage_verdict() { # Written whenever $COV_PCT improves on the baseline beyond the epsilon, or when # the baseline has no coverage key yet. Uses a targeted jq merge — touching only -# .coverage.hermeticLine and .updated — rather than reconstructing the whole file from a -# heredoc the way save_baseline (warnings) does: that approach only knows about -# schema/package/project/testProject/warnings/updated and silently drops any -# other key (mutationScore, e2eHealthUrl, formatBaseRef) whenever it fires. Since -# save_baseline already ran earlier in the same gate.sh invocation (stage_build), -# a merge here avoids stomping whatever it just wrote. +# .coverage.hermeticLine and .updated — the same way save_baseline (warnings) merges +# into an existing file. Since save_baseline already ran earlier in the same gate.sh +# invocation (stage_build), a merge here avoids stomping whatever it just wrote. # # Must be called while $COV_PCT/$COV_COVERED/$COV_VALID still reflect the # HERMETIC report — stage_coverage re-parses the full (incl. E2E) report diff --git a/scripts/quality-gate/lib/warnings.sh b/scripts/quality-gate/lib/warnings.sh index d46f9cf3..22c479b9 100644 --- a/scripts/quality-gate/lib/warnings.sh +++ b/scripts/quality-gate/lib/warnings.sh @@ -51,6 +51,30 @@ warn_verdict() { # warn_verdict fi } +# An existing baseline gets only .warnings and .updated replaced: every other key +# (coverage.hermeticLine, mutationScore, e2eHealthUrl, formatBaseRef) is kept. +# Rewriting the whole file dropped coverage.hermeticLine whenever warnings fell, +# and save_coverage_baseline then re-established it from the current run, up to +# COVERAGE_EPSILON lower: the ratchet slid down a little on every such merge. +merge_warnings_into_baseline() { # merge_warnings_into_baseline + jq --argjson total "$1" --argjson codes "$2" --arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.warnings = {total: $total, byCode: $codes} | .updated = $now' \ + "$BASELINE" > "$BASELINE.tmp" && mv "$BASELINE.tmp" "$BASELINE" +} + +write_new_baseline() { # write_new_baseline + cat > "$BASELINE" <1?",":""), $1, $2} END{print ""}' \ "$OUT/warnings-by-code.txt" | sed 's/^/{/; s/$/}/') - cat > "$BASELINE" <