Improve conformance baseline reporting

.github/workflows/mcp_conformance.yml

@@ -7,6 +7,7 @@ permissions:
   contents: read
 
 env:
+  MCP_CONFORMANCE_COLOR: always
   MCP_CONFORMANCE_VERSION: 0.2.0-alpha.11
   MCP_CONFORMANCE_SOURCE_SHA: c321dd32035556e6769d3724a8ee97d87c3faaac # pragma: allowlist secret
   MCP_CONFORMANCE_SPEC_VERSION: 2026-07-28
@@ -24,6 +25,9 @@ jobs:
       - name: Check out data plane
         uses: actions/checkout@v6.0.2
 
+      - name: Test conformance reporter
+        run: tests/conformance/report-baseline-diff-test.sh
+
       - name: Download the conformance binary
         uses: actions/download-artifact@v8.0.1
         with:
@@ -112,13 +116,3 @@ jobs:
         env:
           MCP_CONFORMANCE_TOKEN: cleanup-only
         run: tests/conformance/stop-live-stack.sh
-
-      - name: Enforce conformance baseline
-        if: always()
-        env:
-          RUNNER_STATUS: ${{ steps.runner.outputs.status }}
-        run: |
-          if [ "${RUNNER_STATUS}" != "0" ]; then
-            echo "::error title=Conformance baseline mismatch::Actual findings differ from the expected baseline. See the baseline diff step and job summary."
-            exit 1
-          fi

Makefile

@@ -7,7 +7,7 @@ ARGS     ?=
 DETECT_SECRETS_SPEC ?= git+https://github.com/ibm/detect-secrets.git@076672a9a01abdfc7ecee2e7d14f08cdccb73976
 DETECT_SECRETS_EXCLUDE := '(?x)(Cargo\.lock$$|\.lock$$)|^\.secrets\.baseline$$'
 
-.PHONY: help docker-prod compose-up compose-down conformance docs-serve pre-commit secrets-scan-all configure-git
+.PHONY: help docker-prod compose-up compose-down conformance conformance-bless docs-serve pre-commit secrets-scan-all configure-git
 
 help: ## Show available commands
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}'
@@ -29,6 +29,9 @@ conformance: ## Build the data plane and run official MCP 2026-07-28 conformance
 	docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile .
 	CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh
 
+conformance-bless: ## Run conformance and update the expected-failure baseline
+	MCP_CONFORMANCE_BLESS=true $(MAKE) conformance
+
 docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000
 	mdbook serve _context/wiki --hostname 127.0.0.1 --port 3000 --open
 

tests/conformance/report-baseline-diff-test.sh

@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+reporter="${script_dir}/report-baseline-diff.sh"
+state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-test.XXXXXX")"
+suite_dir="${state_dir}/suite"
+results_dir="${state_dir}/results"
+baseline_file="${state_dir}/expected-failures.yml"
+upstream_file="${state_dir}/upstream-fixture-failures.yml"
+summary_file="${state_dir}/summary.md"
+
+cleanup() {
+  rm -rf -- "${state_dir}"
+}
+trap cleanup EXIT INT TERM
+
+mkdir -p "${suite_dir}/requirements" "${results_dir}"
+
+cat > "${suite_dir}/requirements/2026-07-28.yaml" <<'EOF'
+server:
+  - expected-check
+  - expected-whole
+  - regression
+  - xpass-check
+  - xpass-whole
+  - absent-check
+  - upstream
+  - duplicate
+  - normal-pass
+EOF
+
+cat > "${baseline_file}" <<'EOF'
+server:
+  - expected-check:known
+  - expected-whole
+  - xpass-check:fixed
+  - xpass-whole
+  - absent-check:not-emitted
+EOF
+
+cat > "${upstream_file}" <<'EOF'
+server:
+  - upstream:fixture-defect
+EOF
+
+write_checks() {
+  scenario="$1"
+  checks="$2"
+  result_dir="${results_dir}/server-${scenario}-2026-08-18T12-00-00-000Z"
+  mkdir -p "${result_dir}"
+  printf '%s\n' "${checks}" > "${result_dir}/checks.json"
+}
+
+write_checks expected-check '[{"id":"known","status":"FAILURE"}]'
+write_checks expected-whole '[{"id":"any-failure","status":"WARNING"}]'
+write_checks regression '[{"id":"new-failure","status":"FAILURE"}]'
+write_checks xpass-check '[{"id":"fixed","status":"SUCCESS"}]'
+write_checks xpass-whole '[{"id":"all-good","status":"SUCCESS"}]'
+write_checks absent-check '[{"id":"other","status":"SUCCESS"}]'
+write_checks upstream '[{"id":"fixture-defect","status":"FAILURE","errorMessage":"must not be reported as a dataplane failure"}]'
+write_checks duplicate '[{"id":"repeated","status":"FAILURE"},{"id":"repeated","status":"SUCCESS"}]'
+write_checks normal-pass '[{"id":"good","status":"SUCCESS"},{"id":"not-applicable","status":"SKIPPED"}]'
+
+assert_contains() {
+  haystack="$1"
+  needle="$2"
+  if [[ "${haystack}" != *"${needle}"* ]]; then
+    echo "Expected output to contain: ${needle}" >&2
+    echo "${haystack}" >&2
+    exit 1
+  fi
+}
+
+assert_not_contains() {
+  haystack="$1"
+  needle="$2"
+  if [[ "${haystack}" == *"${needle}"* ]]; then
+    echo "Expected output not to contain: ${needle}" >&2
+    echo "${haystack}" >&2
+    exit 1
+  fi
+}
+
+set +e
+output="$(
+  GITHUB_ACTIONS=true \
+  GITHUB_STEP_SUMMARY="${summary_file}" \
+  MCP_CONFORMANCE_COLOR=never \
+  MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+    "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" 2>&1
+)"
+status="$?"
+set -e
+
+if [ "${status}" -ne 1 ]; then
+  echo "Expected mismatch status 1, got ${status}" >&2
+  echo "${output}" >&2
+  exit 1
+fi
+
+assert_contains "${output}" 'XFAIL expected-check:known'
+assert_contains "${output}" 'XFAIL expected-whole'
+assert_contains "${output}" 'UPSTREAM upstream:fixture-defect'
+assert_contains "${output}" 'FAIL duplicate:repeated (expected PASS, got FAILURE)'
+assert_contains "${output}" 'FAIL regression:new-failure (expected PASS, got FAILURE)'
+assert_contains "${output}" 'XPASS xpass-check:fixed (expected FAILURE, got PASS)'
+assert_contains "${output}" 'XPASS xpass-whole (expected FAILURE, got PASS)'
+assert_not_contains "${output}" 'XPASS absent-check:not-emitted'
+assert_not_contains "${output}" '::error title=Expected conformance pass failed::upstream:fixture-defect'
+
+summary="$(cat "${summary_file}")"
+assert_contains "${summary}" '| Pinned fixture findings ignored | 1 |'
+assert_not_contains "${summary}" 'upstream:fixture-defect'
+
+cat > "${state_dir}/unmatched-upstream.yml" <<'EOF'
+server:
+  - never-seen:fixture-defect
+EOF
+set +e
+unmatched_upstream_output="$(
+  MCP_CONFORMANCE_COLOR=never \
+  MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+    "${reporter}" \
+      "${results_dir}" \
+      "${baseline_file}" \
+      "${state_dir}/unmatched-upstream.yml" 2>&1
+)"
+unmatched_upstream_status="$?"
+set -e
+if [ "${unmatched_upstream_status}" -ne 1 ]; then
+  echo "Expected unmatched-upstream status 1, got ${unmatched_upstream_status}" >&2
+  exit 1
+fi
+assert_contains "${unmatched_upstream_output}" 'FAIL upstream:fixture-defect'
+
+echo 'server: []' > "${state_dir}/empty-baseline.yml"
+set +e
+empty_baseline_output="$(
+  MCP_CONFORMANCE_COLOR=never \
+  MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+    "${reporter}" \
+      "${results_dir}" \
+      "${state_dir}/empty-baseline.yml" \
+      "${upstream_file}" 2>&1
+)"
+empty_baseline_status="$?"
+set -e
+if [ "${empty_baseline_status}" -ne 1 ]; then
+  echo "Expected empty-baseline status 1, got ${empty_baseline_status}" >&2
+  exit 1
+fi
+assert_contains "${empty_baseline_output}" 'FAIL regression:new-failure'
+
+bless_output="$(
+  MCP_CONFORMANCE_COLOR=never \
+  MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+    "${reporter}" --bless "${results_dir}" "${baseline_file}" "${upstream_file}"
+)"
+assert_contains "${bless_output}" "BLESS updated ${baseline_file}"
+
+cat > "${state_dir}/expected-after-bless.yml" <<'EOF'
+# Generated by `make conformance-bless` from scored dataplane findings.
+# Pinned fixture findings are excluded; see upstream-fixture-failures.yml.
+server:
+  - duplicate:repeated
+  - expected-check:known
+  - expected-whole:any-failure
+  - regression:new-failure
+EOF
+diff -u "${state_dir}/expected-after-bless.yml" "${baseline_file}"
+
+MCP_CONFORMANCE_COLOR=never \
+MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+  "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" > /dev/null
+
+baseline_before_missing="$(cat "${baseline_file}")"
+set +e
+MCP_CONFORMANCE_COLOR=never \
+MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \
+  "${reporter}" --bless "${state_dir}/missing" "${baseline_file}" "${upstream_file}" > /dev/null 2>&1
+missing_status="$?"
+set -e
+
+if [ "${missing_status}" -ne 2 ]; then
+  echo "Expected missing-results status 2, got ${missing_status}" >&2
+  exit 1
+fi
+if [ "$(cat "${baseline_file}")" != "${baseline_before_missing}" ]; then
+  echo 'Bless changed the baseline without results' >&2
+  exit 1
+fi
+
+echo 'conformance reporter tests passed'