fix(quality-gate): stop baselines from loosening

.github/workflows/build-and-test.yml

@@ -104,6 +104,32 @@ jobs:
           body: |
             Refreshes the quality-gate baseline after a merge to master.
 
+  # The gate reads the baseline it is given, so a PR that loosens a
+  # .gate-baseline.json would pass the gate it just weakened.
+  baseline-tightened:
+    name: Baseline only tightens
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Compare each changed baseline with the PR's base
+        run: |
+          verdict=0
+          while IFS= read -r file; do
+            before="$RUNNER_TEMP/before.json"
+            git show "$BASE:$file" > "$before" 2>/dev/null || rm -f "$before"
+            echo "== $file"
+            ./scripts/quality-gate/baseline-tightened.sh "$before" "$file" || verdict=1
+          done < <(git diff --name-only "$BASE"...HEAD -- 'packages/*/.gate-baseline.json')
+          exit "$verdict"
+        env:
+          BASE: ${{ github.event.pull_request.base.sha }}
+
   # Validates the in-code capability annotations against the canonical matrix spec:
   # catches a mistyped feature key or a feature reported with conflicting statuses.
   capability-matrix:

scripts/quality-gate/README.md

@@ -143,6 +143,16 @@ FIXTURE_KEEP=1 ./scripts/quality-gate/fixtures/run-fixtures.sh   # keep work dir
 Change the gate's behavior → add or update the fixture that pins it. A behavior with no
 fixture is untested.
 
+## Baselines only tighten
+
+The gate reads the baseline it is given, so a PR that loosens one would pass the gate it
+just weakened. `baseline-tightened.sh` runs on every PR against each changed
+`.gate-baseline.json` and fails when warnings rose, coverage fell, an identity field
+changed or the file was deleted. The gate itself only ever writes a stricter baseline, so
+a looser one can only come from a hand edit.
+
+```sh
+./scripts/quality-gate/fixtures-baseline-tightened/run.sh   # its cases
 ## Which CI jobs a change needs
 
 `changed-areas.sh` classifies a diff, and each workflow skips the jobs it cannot affect:
@@ -170,7 +180,9 @@ lib/warnings.sh         analyzer warnings + per-package baseline I/O
 lib/coverage.sh         line coverage + per-package baseline I/O
 lib/stages.sh           one definition of each stage, used for both scopes
 format-suggest.sh       prints the format diff for a changed file (dev helper)
-changed-areas.sh        classifies a diff into the areas CI jobs care about
+baseline-tightened.sh   fails when a baseline got looser
 fixtures/               the gate's own test suite
+fixtures-baseline-tightened/  cases for baseline-tightened.sh
 fixtures-changed-areas/ cases for changed-areas.sh
+changed-areas.sh        classifies a diff into the areas CI jobs care about
 ```

scripts/quality-gate/baseline-tightened.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# baseline-tightened.sh <before.json> <after.json> — exit 0 when the per-package
+# .gate-baseline.json only got stricter, 1 when any measurement got looser.
+#
+# The gate reads the baseline it is given, so a PR that loosens one passes the
+# gate it just weakened. This check closes that: the gate itself only ever moves
+# a baseline toward stricter (warnings down, coverage up), so a looser value can
+# only come from a hand edit.
+#
+#   before missing          a new package's first baseline: allowed.
+#   after missing           a deleted baseline: the next run would re-establish it
+#                           from whatever it measures, so this is a loosening.
+#   package/project/...     identity fields must not change.
+#   warnings.total          must not rise.
+#   warnings.byCode.<code>  must not rise; a code absent before counted as 0.
+#   coverage.hermeticLine   must not fall, and must not disappear.
+#   updated                 free.
+#
+set -euo pipefail
+
+[[ $# -eq 2 ]] || { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 3; }
+BEFORE="$1"; AFTER="$2"
+
+command -v jq >/dev/null 2>&1 || { echo "jq not found" >&2; exit 3; }
+
+[[ -f "$BEFORE" ]] || { echo "new baseline: nothing to compare"; exit 0; }
+[[ -f "$AFTER" ]] || { echo "LOOSENED: baseline deleted"; exit 1; }
+
+loosenings="$(jq -rn --slurpfile before "$BEFORE" --slurpfile after "$AFTER" '
+  ($before[0]) as $b | ($after[0]) as $a
+  | [
+      ( ["schema", "package", "project", "testProject"][]
+        | select($b[.] != null and $a[.] != $b[.])
+        | "\(.) changed: \($b[.]) -> \($a[.])" ),
+      ( select($b.warnings.total != null
+               and (($a.warnings.total // infinite) > $b.warnings.total))
+        | "warnings.total rose: \($b.warnings.total) -> \($a.warnings.total)" ),
+      ( ($a.warnings.byCode // {}) | to_entries[]
+        | select(.value > ($b.warnings.byCode[.key] // 0))
+        | "warnings.byCode.\(.key) rose: \($b.warnings.byCode[.key] // 0) -> \(.value)" ),
+      ( select($b.coverage.hermeticLine != null
+               and (($a.coverage.hermeticLine // -1) < $b.coverage.hermeticLine))
+        | "coverage.hermeticLine fell: \($b.coverage.hermeticLine) -> \($a.coverage.hermeticLine)" )
+    ]
+  | .[]
+')"
+
+if [[ -n "$loosenings" ]]; then
+  while IFS= read -r line; do echo "LOOSENED: $line"; done <<<"$loosenings"
+  exit 1
+fi
+echo "tightened or unchanged"

scripts/quality-gate/fixtures-baseline-tightened/run.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# run.sh — the test for baseline-tightened.sh.
+#
+# Each case is inline: a before/after baseline pair and the verdict it must
+# produce. A wrong answer here lets a PR loosen the gate unnoticed, or blocks a
+# legitimate baseline refresh, so every rule in the script's header has a case.
+# A "loosened" verdict must come with a LOOSENED line, so a jq error can never
+# pass for a detected loosening.
+#
+#   run.sh    run every case
+#
+# Exit 0 if every case passed, 1 otherwise.
+
+set -uo pipefail
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TIGHTENED="$HERE/../baseline-tightened.sh"
+RUNROOT="${FIXTURE_RUNROOT:-${TMPDIR:-/tmp}/baseline-tightened-fixtures.$$}"
+
+command -v jq >/dev/null || { echo "jq required" >&2; exit 3; }
+
+if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then
+  RED=$'\033[31m'; GRN=$'\033[32m'; DIM=$'\033[2m'; BLD=$'\033[1m'; OFF=$'\033[0m'
+else RED=""; GRN=""; DIM=""; BLD=""; OFF=""; fi
+
+mkdir -p "$RUNROOT"
+trap '[[ -n "${FIXTURE_KEEP:-}" ]] || rm -rf "$RUNROOT"' EXIT
+
+PASS=0; FAIL=0
+
+baseline_case() {
+  local name="$1" expected="$2" before="$3" after="$4"
+  local dir="$RUNROOT/${name// /-}"
+  mkdir -p "$dir"
+  [[ -n "$before" ]] && printf '%s' "$before" >"$dir/before.json"
+  [[ -n "$after" ]] && printf '%s' "$after" >"$dir/after.json"
+
+  bash "$TIGHTENED" "$dir/before.json" "$dir/after.json" >"$dir/out.log" 2>&1
+  local rc=$?
+  local verdict="error (exit $rc): $(tr '\n' ' ' <"$dir/out.log")"
+  [[ $rc -eq 0 ]] && verdict=tightened
+  [[ $rc -eq 1 ]] && grep -q '^LOOSENED: ' "$dir/out.log" && verdict=loosened
+
+  if [[ "$expected" == "$verdict" ]]; then
+    printf "  ${GRN}PASS${OFF}  %s\n" "$name"; PASS=$((PASS+1))
+  else
+    printf "  ${RED}FAIL${OFF}  %s\n      ${DIM}expected: %s${OFF}\n      ${DIM}actual:   %s${OFF}\n" \
+      "$name" "$expected" "$verdict"
+    FAIL=$((FAIL+1))
+  fi
+}
+
+BASE='{"schema":1,"package":"Gotrue","project":"Gotrue/Gotrue.csproj","testProject":"Gotrue.Tests/Gotrue.Tests.csproj","warnings":{"total":10,"byCode":{"CS1591":6,"CS8618":4}},"updated":"2026-10-01T00:00:00Z","coverage":{"hermeticLine":60.5}}'
+with() { jq -c "$1" <<<"$BASE"; }
+
+baseline_case "unchanged"               tightened "$BASE" "$BASE"
+baseline_case "only updated moved"      tightened "$BASE" "$(with '.updated = "2026-10-30T00:00:00Z"')"
+baseline_case "warnings lowered"        tightened "$BASE" "$(with '.warnings = {"total":8,"byCode":{"CS1591":4,"CS8618":4}}')"
+baseline_case "code dropped to zero"    tightened "$BASE" "$(with '.warnings = {"total":6,"byCode":{"CS1591":6}}')"
+baseline_case "coverage raised"         tightened "$BASE" "$(with '.coverage.hermeticLine = 62.1')"
+baseline_case "coverage added"          tightened "$(with 'del(.coverage)')" "$BASE"
+baseline_case "new package"             tightened "" "$BASE"
+baseline_case "warnings total rose"     loosened  "$BASE" "$(with '.warnings.total = 11')"
+baseline_case "warnings total removed"  loosened  "$BASE" "$(with 'del(.warnings.total)')"
+baseline_case "one code rose"           loosened  "$BASE" "$(with '.warnings.byCode.CS8618 = 5')"
+baseline_case "new code appeared"       loosened  "$BASE" "$(with '.warnings.byCode.CS0168 = 1')"
+baseline_case "coverage fell"           loosened  "$BASE" "$(with '.coverage.hermeticLine = 60.4')"
+baseline_case "coverage removed"        loosened  "$BASE" "$(with 'del(.coverage)')"
+baseline_case "project changed"         loosened  "$BASE" "$(with '.project = "Other/Other.csproj"')"
+baseline_case "baseline deleted"        loosened  "$BASE" ""
+
+echo
+if [[ $FAIL -eq 0 ]]; then
+  echo "${GRN}${BLD}cases: $PASS/$((PASS+FAIL)) passed${OFF}"; exit 0
+else
+  echo "${RED}${BLD}cases: $FAIL/$((PASS+FAIL)) failed${OFF}"; exit 1
+fi

scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/expect.json

@@ -0,0 +1,11 @@
+{
+  "description": "baseline-overwrite with a coverage baseline already recorded: lowering warnings must rewrite only .warnings and .updated. Guards the ratchet slide where rewriting the whole file dropped coverage.hermeticLine and the next stage re-established it lower.",
+  "mode": "--fast --overwrite-baseline",
+  "exit": 0,
+  "verdict": "PARTIAL",
+  "baselineWrite": true,
+  "baselineKeeps": [".coverage.hermeticLine"],
+  "stages": [
+    { "id": "1a", "status": "PASS" }
+  ]
+}

scripts/quality-gate/fixtures/baseline-overwrite-keeps-coverage/overlay/.gate-baseline.json

@@ -0,0 +1,9 @@
+{
+  "schema": 1,
+  "package": "baseline-overwrite-keeps-coverage",
+  "project": "Sample/Sample.csproj",
+  "testProject": "Sample.Tests/Sample.Tests.csproj",
+  "warnings": { "total": 3, "byCode": {} },
+  "updated": "2026-08-12T00:00:00Z",
+  "coverage": { "hermeticLine": 99.5 }
+}

scripts/quality-gate/fixtures/run-fixtures.sh

@@ -155,6 +155,14 @@ for fx in "${fixtures[@]}"; do
     fi
   fi
 
+  # Optional: keys a baseline write must leave as they were (expect.baselineKeeps: [jq paths]).
+  while IFS= read -r path; do
+    [[ -z "$path" ]] && continue
+    was="$(jq -c "$path" <<<"$bl_before")"
+    now="$(jq -c "$path" "$bl" 2>/dev/null || echo __ABSENT__)"
+    assert "baseline $path" "$was" "$now"
+  done < <(jq -r '.baselineKeeps // [] | .[]' "$exp")
+
   if [[ -z "$miss" ]]; then
     printf "  ${GRN}PASS${OFF}  %-18s ${DIM}%s${OFF}\n" "$name" "$(jq -r '.description // ""' "$exp")"
     PASS=$((PASS+1))

scripts/quality-gate/lib/coverage.sh

@@ -83,12 +83,9 @@ coverage_verdict() {
 
 # Written whenever $COV_PCT improves on the baseline beyond the epsilon, or when
 # the baseline has no coverage key yet. Uses a targeted jq merge — touching only
-# .coverage.hermeticLine and .updated — rather than reconstructing the whole file from a
-# heredoc the way save_baseline (warnings) does: that approach only knows about
-# schema/package/project/testProject/warnings/updated and silently drops any
-# other key (mutationScore, e2eHealthUrl, formatBaseRef) whenever it fires. Since
-# save_baseline already ran earlier in the same gate.sh invocation (stage_build),
-# a merge here avoids stomping whatever it just wrote.
+# .coverage.hermeticLine and .updated — the same way save_baseline (warnings) merges
+# into an existing file. Since save_baseline already ran earlier in the same gate.sh
+# invocation (stage_build), a merge here avoids stomping whatever it just wrote.
 #
 # Must be called while $COV_PCT/$COV_COVERED/$COV_VALID still reflect the
 # HERMETIC report — stage_coverage re-parses the full (incl. E2E) report

scripts/quality-gate/lib/warnings.sh

@@ -51,6 +51,30 @@ warn_verdict() {  # warn_verdict <codes-file>
   fi
 }
 
+# An existing baseline gets only .warnings and .updated replaced: every other key
+# (coverage.hermeticLine, mutationScore, e2eHealthUrl, formatBaseRef) is kept.
+# Rewriting the whole file dropped coverage.hermeticLine whenever warnings fell,
+# and save_coverage_baseline then re-established it from the current run, up to
+# COVERAGE_EPSILON lower: the ratchet slid down a little on every such merge.
+merge_warnings_into_baseline() {  # merge_warnings_into_baseline <total> <byCode-json>
+  jq --argjson total "$1" --argjson codes "$2" --arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
+    '.warnings = {total: $total, byCode: $codes} | .updated = $now' \
+    "$BASELINE" > "$BASELINE.tmp" && mv "$BASELINE.tmp" "$BASELINE"
+}
+
+write_new_baseline() {  # write_new_baseline <total> <byCode-json>
+  cat > "$BASELINE" <<EOF
+{
+  "schema": 1,
+  "package": "$PACKAGE_NAME",
+  "project": "${PROJECT#$PACKAGE_DIR/}",
+  "testProject": "${TEST_PROJECT#$PACKAGE_DIR/}",
+  "warnings": { "total": $1, "byCode": $2 },
+  "updated": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+}
+EOF
+}
+
 # Written whenever a measurement improves, or when the file doesn't exist yet.
 # Reads $BASELINE/$PACKAGE_NAME/$PROJECT/$TEST_PROJECT/$PACKAGE_DIR/$OUT — all set
 # by _use_pkg before this is called.
@@ -66,16 +90,11 @@ save_baseline() {
     '{printf "%s\"%s\":%s", (NR>1?",":""), $1, $2} END{print ""}' \
     "$OUT/warnings-by-code.txt" | sed 's/^/{/; s/$/}/')
 
-  cat > "$BASELINE" <<EOF
-{
-  "schema": 1,
-  "package": "$PACKAGE_NAME",
-  "project": "${PROJECT#$PACKAGE_DIR/}",
-  "testProject": "${TEST_PROJECT#$PACKAGE_DIR/}",
-  "warnings": { "total": $w, "byCode": $codes },
-  "updated": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-}
-EOF
+  if [[ -f "$BASELINE" ]]; then
+    merge_warnings_into_baseline "$w" "$codes"
+  else
+    write_new_baseline "$w" "$codes"
+  fi
   if [[ -z "$base_w" ]]; then
     echo "${GRN}baseline created${OFF} $BASELINE ${DIM}— commit it${OFF}"
   else