diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml index 333c810..36957fd 100644 --- a/.github/workflows/mcp_conformance.yml +++ b/.github/workflows/mcp_conformance.yml @@ -7,6 +7,7 @@ permissions: contents: read env: + MCP_CONFORMANCE_COLOR: always MCP_CONFORMANCE_VERSION: 0.2.0-alpha.11 MCP_CONFORMANCE_SOURCE_SHA: c321dd32035556e6769d3724a8ee97d87c3faaac # pragma: allowlist secret MCP_CONFORMANCE_SPEC_VERSION: 2026-07-28 @@ -24,6 +25,9 @@ jobs: - name: Check out data plane uses: actions/checkout@v6.0.2 + - name: Test conformance reporter + run: tests/conformance/report-baseline-diff-test.sh + - name: Download the conformance binary uses: actions/download-artifact@v8.0.1 with: @@ -112,13 +116,3 @@ jobs: env: MCP_CONFORMANCE_TOKEN: cleanup-only run: tests/conformance/stop-live-stack.sh - - - name: Enforce conformance baseline - if: always() - env: - RUNNER_STATUS: ${{ steps.runner.outputs.status }} - run: | - if [ "${RUNNER_STATUS}" != "0" ]; then - echo "::error title=Conformance baseline mismatch::Actual findings differ from the expected baseline. See the baseline diff step and job summary." - exit 1 - fi diff --git a/Makefile b/Makefile index 4357644..5f3ea13 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ ARGS ?= DETECT_SECRETS_SPEC ?= git+https://github.com/ibm/detect-secrets.git@076672a9a01abdfc7ecee2e7d14f08cdccb73976 DETECT_SECRETS_EXCLUDE := '(?x)(Cargo\.lock$$|\.lock$$)|^\.secrets\.baseline$$' -.PHONY: help docker-prod compose-up compose-down conformance docs-serve pre-commit secrets-scan-all configure-git +.PHONY: help docker-prod compose-up compose-down conformance conformance-bless docs-serve pre-commit secrets-scan-all configure-git help: ## Show available commands @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}' @@ -29,6 +29,9 @@ conformance: ## Build the data plane and run official MCP 2026-07-28 conformance docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/conformance.Dockerfile . CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh +conformance-bless: ## Run conformance and update the expected-failure baseline + MCP_CONFORMANCE_BLESS=true $(MAKE) conformance + docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000 mdbook serve _context/wiki --hostname 127.0.0.1 --port 3000 --open diff --git a/tests/conformance/report-baseline-diff-test.sh b/tests/conformance/report-baseline-diff-test.sh new file mode 100755 index 0000000..525f357 --- /dev/null +++ b/tests/conformance/report-baseline-diff-test.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +reporter="${script_dir}/report-baseline-diff.sh" +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-test.XXXXXX")" +suite_dir="${state_dir}/suite" +results_dir="${state_dir}/results" +baseline_file="${state_dir}/expected-failures.yml" +upstream_file="${state_dir}/upstream-fixture-failures.yml" +summary_file="${state_dir}/summary.md" + +cleanup() { + rm -rf -- "${state_dir}" +} +trap cleanup EXIT INT TERM + +mkdir -p "${suite_dir}/requirements" "${results_dir}" + +cat > "${suite_dir}/requirements/2026-07-28.yaml" <<'EOF' +server: + - expected-check + - expected-whole + - regression + - xpass-check + - xpass-whole + - absent-check + - upstream + - duplicate + - normal-pass +EOF + +cat > "${baseline_file}" <<'EOF' +server: + - expected-check:known + - expected-whole + - xpass-check:fixed + - xpass-whole + - absent-check:not-emitted +EOF + +cat > "${upstream_file}" <<'EOF' +server: + - upstream:fixture-defect +EOF + +write_checks() { + scenario="$1" + checks="$2" + result_dir="${results_dir}/server-${scenario}-2026-08-18T12-00-00-000Z" + mkdir -p "${result_dir}" + printf '%s\n' "${checks}" > "${result_dir}/checks.json" +} + +write_checks expected-check '[{"id":"known","status":"FAILURE"}]' +write_checks expected-whole '[{"id":"any-failure","status":"WARNING"}]' +write_checks regression '[{"id":"new-failure","status":"FAILURE"}]' +write_checks xpass-check '[{"id":"fixed","status":"SUCCESS"}]' +write_checks xpass-whole '[{"id":"all-good","status":"SUCCESS"}]' +write_checks absent-check '[{"id":"other","status":"SUCCESS"}]' +write_checks upstream '[{"id":"fixture-defect","status":"FAILURE","errorMessage":"must not be reported as a dataplane failure"}]' +write_checks duplicate '[{"id":"repeated","status":"FAILURE"},{"id":"repeated","status":"SUCCESS"}]' +write_checks normal-pass '[{"id":"good","status":"SUCCESS"},{"id":"not-applicable","status":"SKIPPED"}]' + +assert_contains() { + haystack="$1" + needle="$2" + if [[ "${haystack}" != *"${needle}"* ]]; then + echo "Expected output to contain: ${needle}" >&2 + echo "${haystack}" >&2 + exit 1 + fi +} + +assert_not_contains() { + haystack="$1" + needle="$2" + if [[ "${haystack}" == *"${needle}"* ]]; then + echo "Expected output not to contain: ${needle}" >&2 + echo "${haystack}" >&2 + exit 1 + fi +} + +set +e +output="$( + GITHUB_ACTIONS=true \ + GITHUB_STEP_SUMMARY="${summary_file}" \ + MCP_CONFORMANCE_COLOR=never \ + MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" 2>&1 +)" +status="$?" +set -e + +if [ "${status}" -ne 1 ]; then + echo "Expected mismatch status 1, got ${status}" >&2 + echo "${output}" >&2 + exit 1 +fi + +assert_contains "${output}" 'XFAIL expected-check:known' +assert_contains "${output}" 'XFAIL expected-whole' +assert_contains "${output}" 'UPSTREAM upstream:fixture-defect' +assert_contains "${output}" 'FAIL duplicate:repeated (expected PASS, got FAILURE)' +assert_contains "${output}" 'FAIL regression:new-failure (expected PASS, got FAILURE)' +assert_contains "${output}" 'XPASS xpass-check:fixed (expected FAILURE, got PASS)' +assert_contains "${output}" 'XPASS xpass-whole (expected FAILURE, got PASS)' +assert_not_contains "${output}" 'XPASS absent-check:not-emitted' +assert_not_contains "${output}" '::error title=Expected conformance pass failed::upstream:fixture-defect' + +summary="$(cat "${summary_file}")" +assert_contains "${summary}" '| Pinned fixture findings ignored | 1 |' +assert_not_contains "${summary}" 'upstream:fixture-defect' + +cat > "${state_dir}/unmatched-upstream.yml" <<'EOF' +server: + - never-seen:fixture-defect +EOF +set +e +unmatched_upstream_output="$( + MCP_CONFORMANCE_COLOR=never \ + MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" \ + "${results_dir}" \ + "${baseline_file}" \ + "${state_dir}/unmatched-upstream.yml" 2>&1 +)" +unmatched_upstream_status="$?" +set -e +if [ "${unmatched_upstream_status}" -ne 1 ]; then + echo "Expected unmatched-upstream status 1, got ${unmatched_upstream_status}" >&2 + exit 1 +fi +assert_contains "${unmatched_upstream_output}" 'FAIL upstream:fixture-defect' + +echo 'server: []' > "${state_dir}/empty-baseline.yml" +set +e +empty_baseline_output="$( + MCP_CONFORMANCE_COLOR=never \ + MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" \ + "${results_dir}" \ + "${state_dir}/empty-baseline.yml" \ + "${upstream_file}" 2>&1 +)" +empty_baseline_status="$?" +set -e +if [ "${empty_baseline_status}" -ne 1 ]; then + echo "Expected empty-baseline status 1, got ${empty_baseline_status}" >&2 + exit 1 +fi +assert_contains "${empty_baseline_output}" 'FAIL regression:new-failure' + +bless_output="$( + MCP_CONFORMANCE_COLOR=never \ + MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" --bless "${results_dir}" "${baseline_file}" "${upstream_file}" +)" +assert_contains "${bless_output}" "BLESS updated ${baseline_file}" + +cat > "${state_dir}/expected-after-bless.yml" <<'EOF' +# Generated by `make conformance-bless` from scored dataplane findings. +# Pinned fixture findings are excluded; see upstream-fixture-failures.yml. +server: + - duplicate:repeated + - expected-check:known + - expected-whole:any-failure + - regression:new-failure +EOF +diff -u "${state_dir}/expected-after-bless.yml" "${baseline_file}" + +MCP_CONFORMANCE_COLOR=never \ +MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" "${results_dir}" "${baseline_file}" "${upstream_file}" > /dev/null + +baseline_before_missing="$(cat "${baseline_file}")" +set +e +MCP_CONFORMANCE_COLOR=never \ +MCP_CONFORMANCE_SUITE_DIR="${suite_dir}" \ + "${reporter}" --bless "${state_dir}/missing" "${baseline_file}" "${upstream_file}" > /dev/null 2>&1 +missing_status="$?" +set -e + +if [ "${missing_status}" -ne 2 ]; then + echo "Expected missing-results status 2, got ${missing_status}" >&2 + exit 1 +fi +if [ "$(cat "${baseline_file}")" != "${baseline_before_missing}" ]; then + echo 'Bless changed the baseline without results' >&2 + exit 1 +fi + +echo 'conformance reporter tests passed' diff --git a/tests/conformance/report-baseline-diff.sh b/tests/conformance/report-baseline-diff.sh index 0656deb..6f7eeb9 100755 --- a/tests/conformance/report-baseline-diff.sh +++ b/tests/conformance/report-baseline-diff.sh @@ -1,6 +1,32 @@ #!/usr/bin/env bash set -euo pipefail +usage() { + cat <<'EOF' +Usage: report-baseline-diff.sh [--bless] [results-dir [baseline-file [upstream-file]]] + +Compare scored MCP conformance checks with the expected-failure baseline. +With --bless, replace the baseline with the current dataplane-owned findings. +EOF +} + +bless=false +case "${1:-}" in + --bless) + bless=true + shift + ;; + --help|-h) + usage + exit 0 + ;; +esac + +if [ "$#" -gt 3 ]; then + usage >&2 + exit 2 +fi + script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "${script_dir}/../.." && pwd)" results_dir="${1:-${repo_root}/conformance-results}" @@ -10,40 +36,97 @@ suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" spec_version="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" requirements_file="${suite_dir}/requirements/${spec_version}.yaml" -for command in awk cut find grep jq sed sort wc; do +for command in awk cmp cp cut find grep jq sed sort wc; do if ! command -v "${command}" > /dev/null 2>&1; then echo "Required command not found: ${command}" >&2 - exit 1 + exit 2 fi done for required_file in "${baseline_file}" "${upstream_file}" "${requirements_file}"; do if [ ! -f "${required_file}" ]; then echo "Required conformance file not found: ${required_file}" >&2 - exit 1 + exit 2 fi done +color_mode="${MCP_CONFORMANCE_COLOR:-${CARGO_TERM_COLOR:-auto}}" +case "${color_mode}" in + always) + use_color=true + ;; + never) + use_color=false + ;; + auto) + if [ -t 1 ] && [ "${TERM:-}" != "dumb" ] && [ -z "${NO_COLOR:-}" ]; then + use_color=true + else + use_color=false + fi + ;; + *) + echo "MCP_CONFORMANCE_COLOR must be auto, always, or never; got: ${color_mode}" >&2 + exit 2 + ;; +esac + +if [ -n "${NO_COLOR:-}" ]; then + use_color=false +fi + +if ${use_color}; then + bold=$'\033[1m' + dim=$'\033[2m' + red=$'\033[31m' + green=$'\033[32m' + yellow=$'\033[33m' + cyan=$'\033[36m' + reset=$'\033[0m' +else + bold="" + dim="" + red="" + green="" + yellow="" + cyan="" + reset="" +fi + state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-diff.XXXXXX")" +actual_checks="${state_dir}/actual-checks.tsv" actual_findings="${state_dir}/actual-findings.tsv" actual_keys="${state_dir}/actual-keys.txt" baseline_entries="${state_dir}/baseline-entries.txt" upstream_entries="${state_dir}/upstream-entries.txt" scored_scenarios="${state_dir}/scored-scenarios.txt" +executed_scenarios="${state_dir}/executed-scenarios.txt" +owned_findings="${state_dir}/owned-findings.txt" +expected_entries="${state_dir}/expected-entries.txt" unexpected_entries="${state_dir}/unexpected-entries.txt" stale_entries="${state_dir}/stale-entries.txt" upstream_matches="${state_dir}/upstream-matches.txt" +passing_checks="${state_dir}/passing-checks.txt" +skipped_checks="${state_dir}/skipped-checks.txt" +baseline_candidate="${state_dir}/expected-failures.yml" cleanup() { rm -f -- \ + "${actual_checks}" \ "${actual_findings}" \ "${actual_keys}" \ "${baseline_entries}" \ "${upstream_entries}" \ "${scored_scenarios}" \ + "${executed_scenarios}" \ + "${owned_findings}" \ + "${expected_entries}" \ "${unexpected_entries}" \ "${stale_entries}" \ - "${upstream_matches}" + "${upstream_matches}" \ + "${passing_checks}" \ + "${skipped_checks}" \ + "${baseline_candidate}" rmdir -- "${state_dir}" } trap cleanup EXIT INT TERM @@ -59,6 +142,35 @@ read_baseline() { ' "$1" | LC_ALL=C sort -u } +line_count() { + wc -l < "$1" | tr -d '[:space:]' +} + +print_row() { + local color="$1" + local label="$2" + local message="$3" + printf ' %b%10s%b %s\n' "${color}${bold}" "${label}" "${reset}" "${message}" +} + +emit_error() { + local title="$1" + local message="$2" + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::error title=${title}::${message}" + fi +} + +write_missing_results_summary() { + local message="$1" + print_row "${red}" "ERROR" "${message}" + emit_error "Conformance results missing" "${message}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '## MCP %s conformance\n\n❌ %s\n' "${spec_version}" "${message}" \ + >> "${GITHUB_STEP_SUMMARY}" + fi +} + awk ' /^server:$/ { in_server = 1; next } in_server && /^[^[:space:]]/ { exit } @@ -72,22 +184,16 @@ awk ' read_baseline "${baseline_file}" > "${baseline_entries}" read_baseline "${upstream_file}" > "${upstream_entries}" +printf '\n%bMCP conformance%b %b(%s)%b\n' "${bold}" "${reset}" "${dim}" "${spec_version}" "${reset}" + if [ ! -d "${results_dir}" ]; then - echo "::warning title=Conformance results missing::No results directory: ${results_dir}" - if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - printf '## Conformance baseline diff\n\nNo conformance results were produced.\n' \ - >> "${GITHUB_STEP_SUMMARY}" - fi - exit 0 + write_missing_results_summary "No results directory: ${results_dir}" + exit 2 fi -: > "${actual_findings}" +: > "${actual_checks}" +: > "${executed_scenarios}" while IFS= read -r -d '' checks_file; do - case "${checks_file}" in - */checks.json) ;; - *) continue ;; - esac - result_name="$(basename -- "$(dirname -- "${checks_file}")")" if [[ ! "${result_name}" =~ ^server-(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z$ ]]; then echo "Skipping unrecognized result directory: ${result_name}" >&2 @@ -95,124 +201,233 @@ while IFS= read -r -d '' checks_file; do fi scenario="${BASH_REMATCH[1]}" - if ! grep --fixed-strings --line-regexp --quiet "${scenario}" "${scored_scenarios}"; then + if ! grep --fixed-strings --line-regexp --quiet -- "${scenario}" "${scored_scenarios}"; then continue fi + echo "${scenario}" >> "${executed_scenarios}" jq --raw-output --arg scenario "${scenario}" ' - .[] | - select(.status == "FAILURE" or .status == "WARNING") | - [($scenario + ":" + .id), .status, (.errorMessage // "")] | + def severity($status): + if $status == "FAILURE" then 3 + elif $status == "WARNING" then 2 + elif $status == "SUCCESS" then 1 + else 0 + end; + + reduce (.[] | select(.status != "INFO")) as $check + ({}; + ($check.id) as $id | + if .[$id] == null or severity($check.status) >= severity(.[$id].status) + then .[$id] = $check + else . + end) | + to_entries[] | + [($scenario + ":" + .key), .value.status, (.value.errorMessage // "")] | @tsv - ' "${checks_file}" >> "${actual_findings}" + ' "${checks_file}" >> "${actual_checks}" done < <(find "${results_dir}" -type f -name checks.json -print0) -LC_ALL=C sort -u -o "${actual_findings}" "${actual_findings}" +LC_ALL=C sort -u -o "${executed_scenarios}" "${executed_scenarios}" +if [ ! -s "${executed_scenarios}" ]; then + write_missing_results_summary "No scored conformance results were found in ${results_dir}" + exit 2 +fi + +LC_ALL=C sort -u -o "${actual_checks}" "${actual_checks}" +awk -F '\t' '$2 == "FAILURE" || $2 == "WARNING"' "${actual_checks}" > "${actual_findings}" cut -f 1 "${actual_findings}" | LC_ALL=C sort -u > "${actual_keys}" +awk -F '\t' '$2 == "SUCCESS" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${passing_checks}" +awk -F '\t' '$2 == "SKIPPED" { print $1 }' "${actual_checks}" | LC_ALL=C sort -u > "${skipped_checks}" -awk ' - NR == FNR { expected[$1] = 1; next } +# Pinned fixture findings are informational: they neither satisfy the dataplane +# baseline nor count as unexpected dataplane failures. +awk -v upstream_file="${upstream_entries}" ' + FILENAME == upstream_file { + upstream[$1] = 1 + next + } { scenario = $1 sub(/:.*/, "", scenario) - if (!(($1 in expected) || (scenario in expected))) print $1 + if (($1 in upstream) || (scenario in upstream)) print $1 } -' "${baseline_entries}" "${actual_keys}" > "${unexpected_entries}" +' "${upstream_entries}" "${actual_keys}" | LC_ALL=C sort -u > "${upstream_matches}" -awk ' - NR == FNR { - actual[$1] = 1 +awk -v upstream_file="${upstream_matches}" ' + FILENAME == upstream_file { upstream[$1] = 1; next } + !($1 in upstream) { print $1 } +' "${upstream_matches}" "${actual_keys}" > "${owned_findings}" + +bless_changed=false +if ${bless}; then + { + # shellcheck disable=SC2016 # Backticks are literal Markdown. + echo '# Generated by `make conformance-bless` from scored dataplane findings.' + echo '# Pinned fixture findings are excluded; see upstream-fixture-failures.yml.' + if [ -s "${owned_findings}" ]; then + echo 'server:' + sed 's/^/ - /' "${owned_findings}" + else + echo 'server: []' + fi + } > "${baseline_candidate}" + + if ! cmp --silent "${owned_findings}" "${baseline_entries}"; then + cp "${baseline_candidate}" "${baseline_file}" + bless_changed=true + fi + read_baseline "${baseline_file}" > "${baseline_entries}" +fi + +# Match actual owned findings against exact or whole-scenario baseline entries. +awk -v baseline_file="${baseline_entries}" ' + FILENAME == baseline_file { + baseline[$1] = 1 + if (index($1, ":") == 0) whole[$1] = 1 + next + } + { scenario = $1 sub(/:.*/, "", scenario) - actual_scenario[scenario] = 1 - next + if (scenario in whole) matched[scenario] = 1 + else if ($1 in baseline) matched[$1] = 1 } - !(($1 in actual) || ($1 in actual_scenario)) { print $1 } -' "${actual_keys}" "${baseline_entries}" > "${stale_entries}" + END { + for (entry in matched) print entry + } +' "${baseline_entries}" "${owned_findings}" | LC_ALL=C sort -u > "${expected_entries}" -awk ' - NR == FNR { upstream[$1] = 1; next } +awk -v baseline_file="${baseline_entries}" ' + FILENAME == baseline_file { + baseline[$1] = 1 + if (index($1, ":") == 0) whole[$1] = 1 + next + } { scenario = $1 sub(/:.*/, "", scenario) - if (($1 in upstream) || (scenario in upstream)) print $1 + if (!(($1 in baseline) || (scenario in whole))) print $1 } -' "${upstream_entries}" "${actual_keys}" > "${upstream_matches}" +' "${baseline_entries}" "${owned_findings}" > "${unexpected_entries}" -line_count() { - wc -l < "$1" | tr -d '[:space:]' -} +# An exact baseline entry is stale only after a demonstrated SUCCESS. Missing +# and SKIPPED checks carry no pass signal. A whole-scenario entry is stale when +# the scenario ran without any dataplane-owned finding. +awk -F '\t' \ + -v checks_file="${actual_checks}" \ + -v scenarios_file="${executed_scenarios}" \ + -v findings_file="${owned_findings}" ' + FILENAME == checks_file { status[$1] = $2; next } + FILENAME == scenarios_file { executed[$1] = 1; next } + FILENAME == findings_file { + scenario = $1 + sub(/:.*/, "", scenario) + failed[scenario] = 1 + next + } + index($1, ":") == 0 { + if (($1 in executed) && !($1 in failed)) print $1 + next + } + status[$1] == "SUCCESS" { print $1 } + ' "${actual_checks}" "${executed_scenarios}" "${owned_findings}" "${baseline_entries}" \ + | LC_ALL=C sort -u > "${stale_entries}" -actual_count="$(line_count "${actual_keys}")" -baseline_count="$(line_count "${baseline_entries}")" +pass_count="$(line_count "${passing_checks}")" +skip_count="$(line_count "${skipped_checks}")" +expected_count="$(line_count "${expected_entries}")" unexpected_count="$(line_count "${unexpected_entries}")" stale_count="$(line_count "${stale_entries}")" upstream_count="$(line_count "${upstream_matches}")" -echo "Conformance baseline diff (${spec_version})" -echo " Actual scored findings: ${actual_count}" -echo " Expected baseline entries: ${baseline_count}" -echo " Matched pinned-fixture findings: ${upstream_count}" -echo " Unexpected findings: ${unexpected_count}" -echo " Stale baseline entries: ${stale_count}" - -if [ "${unexpected_count}" -gt 0 ]; then - echo - echo "Unexpected findings (actual but not in baseline):" - while IFS= read -r key; do - detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")" - echo " - ${key} — ${detail}" - echo "::error title=Unexpected conformance finding::${key}" - done < "${unexpected_entries}" -fi +print_row "${green}" "PASS" "${pass_count} scored checks passed" -if [ "${stale_count}" -gt 0 ]; then - echo - echo "Stale baseline entries (expected but now passing):" - while IFS= read -r key; do - echo " - ${key}" - echo "::error title=Stale conformance baseline::${key} is now passing" - done < "${stale_entries}" +while IFS= read -r key; do + [ -n "${key}" ] || continue + print_row "${yellow}" "XFAIL" "${key} ${dim}(expected failure reproduced)${reset}" +done < "${expected_entries}" + +while IFS= read -r key; do + [ -n "${key}" ] || continue + print_row "${cyan}" "UPSTREAM" "${key} ${dim}(ignored pinned-fixture finding)${reset}" +done < "${upstream_matches}" + +while IFS= read -r key; do + [ -n "${key}" ] || continue + status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" + print_row "${red}" "FAIL" "${key} ${dim}(expected PASS, got ${status})${reset}" + emit_error "Expected conformance pass failed" "${key}" +done < "${unexpected_entries}" + +while IFS= read -r key; do + [ -n "${key}" ] || continue + print_row "${red}" "XPASS" "${key} ${dim}(expected FAILURE, got PASS)${reset}" + emit_error "Expected conformance failure passed" "${key}" +done < "${stale_entries}" + +if [ "${skip_count}" -gt 0 ]; then + print_row "${dim}" "SKIP" "${skip_count} scored checks skipped" fi -if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then - echo "Actual scored findings match the expected baseline." +if ${bless}; then + if ${bless_changed}; then + print_row "${green}" "BLESS" "updated ${baseline_file}" + else + print_row "${green}" "BLESS" "${baseline_file} was already current" + fi fi +printf '\n%bSummary%b: %s passed, %s expected failures, %s upstream findings ignored, %s failed, %s unexpected passes\n' \ + "${bold}" "${reset}" \ + "${pass_count}" "${expected_count}" "${upstream_count}" "${unexpected_count}" "${stale_count}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then { - echo "## Conformance baseline diff" - echo - echo "| Category | Count |" - echo "| --- | ---: |" - echo "| Actual scored findings | ${actual_count} |" - echo "| Expected baseline entries | ${baseline_count} |" - echo "| Matched pinned-fixture findings | ${upstream_count} |" - echo "| Unexpected findings | ${unexpected_count} |" - echo "| Stale baseline entries | ${stale_count} |" + echo "## MCP ${spec_version} conformance" echo - echo "The pinned alpha.11 fixture has 7 scored failures and 1 warning recorded in \`upstream-fixture-failures.yml\`; its other 47 failures are extension or pending scenarios and are unscored." + echo '| Outcome | Count |' + echo '| --- | ---: |' + echo "| Scored checks passed | ${pass_count} |" + echo "| Expected failures reproduced | ${expected_count} |" + echo "| Pinned fixture findings ignored | ${upstream_count} |" + echo "| Expected pass, got failure | ${unexpected_count} |" + echo "| Expected failure, got pass | ${stale_count} |" + echo "| Skipped checks | ${skip_count} |" if [ "${unexpected_count}" -gt 0 ]; then echo - echo "### Unexpected findings" + echo '### Expected pass, got failure' while IFS= read -r key; do - detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")" - echo "- \`${key}\` — ${detail}" + [ -n "${key}" ] || continue + status="$(awk -F '\t' -v key="${key}" '$1 == key { print $2; exit }' "${actual_findings}")" + echo "- \`${key}\` — ${status}" done < "${unexpected_entries}" fi if [ "${stale_count}" -gt 0 ]; then echo - echo "### Stale baseline entries" + echo '### Expected failure, got pass' while IFS= read -r key; do + [ -n "${key}" ] || continue echo "- \`${key}\`" done < "${stale_entries}" fi - if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then - echo - echo "Actual scored findings match the expected baseline." + echo + if ${bless}; then + echo "✅ Expected-failure baseline updated with ${expected_count} dataplane findings." + elif [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then + echo '✅ Actual dataplane findings match the expected-failure baseline.' + else + echo '❌ Actual dataplane findings do not match the expected-failure baseline.' fi } >> "${GITHUB_STEP_SUMMARY}" fi + +if ${bless}; then + exit 0 +fi + +if [ "${unexpected_count}" -gt 0 ] || [ "${stale_count}" -gt 0 ]; then + exit 1 +fi diff --git a/tests/conformance/run-conformance.sh b/tests/conformance/run-conformance.sh index 8302110..1727a03 100755 --- a/tests/conformance/run-conformance.sh +++ b/tests/conformance/run-conformance.sh @@ -9,6 +9,7 @@ script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd -- "${script_dir}/../.." && pwd)" suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" conformance_port="${MCP_CONFORMANCE_PORT:-8080}" +results_dir="${MCP_CONFORMANCE_RESULTS_DIR:-${repo_root}/conformance-results}" set +e ( @@ -18,7 +19,7 @@ set +e --url "http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" \ --requirements "${MCP_CONFORMANCE_SPEC_VERSION}" \ --expected-failures "${script_dir}/expected-failures.yml" \ - --output-dir "${repo_root}/conformance-results" + --output-dir "${results_dir}" ) runner_status="$?" set -e diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh index 77c5cbb..72bb5e1 100755 --- a/tests/conformance/run-local.sh +++ b/tests/conformance/run-local.sh @@ -12,6 +12,7 @@ export MCP_CONFORMANCE_SERVER_ID="${MCP_CONFORMANCE_SERVER_ID:-3f33286667d34b65a export MCP_CONFORMANCE_SUITE_DIR="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" export CF_CONTROLPLANE_IMAGE="${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest}" export CF_DATAPLANE_IMAGE="${CF_DATAPLANE_IMAGE:-contextforge-data-plane:conformance}" +export MCP_CONFORMANCE_COLOR="${MCP_CONFORMANCE_COLOR:-auto}" for command in curl docker git jq node npm; do if ! command -v "${command}" > /dev/null 2>&1; then @@ -52,6 +53,9 @@ state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-conformance.XXXXXX")" export GITHUB_ENV="${state_dir}/github-env" export GITHUB_OUTPUT="${state_dir}/github-output" touch "${GITHUB_ENV}" "${GITHUB_OUTPUT}" +mkdir -p "${repo_root}/conformance-results" +export MCP_CONFORMANCE_RESULTS_DIR +MCP_CONFORMANCE_RESULTS_DIR="$(mktemp -d "${repo_root}/conformance-results/run.XXXXXX")" # shellcheck disable=SC2329 # Invoked by the trap below. cleanup() { @@ -98,4 +102,17 @@ if [ -z "${runner_status}" ]; then echo "Conformance runner did not report a status." >&2 exit 1 fi -exit "${runner_status}" + +set +e +if [ "${MCP_CONFORMANCE_BLESS:-false}" = "true" ]; then + "${script_dir}/report-baseline-diff.sh" --bless "${MCP_CONFORMANCE_RESULTS_DIR}" +else + "${script_dir}/report-baseline-diff.sh" "${MCP_CONFORMANCE_RESULTS_DIR}" +fi +report_status="$?" +set -e + +if [ "${runner_status}" -ne 0 ] && [ "${report_status}" -eq 0 ]; then + echo "Official runner status ${runner_status} contained no dataplane baseline mismatch." +fi +exit "${report_status}"