diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml index bf67523..333c810 100644 --- a/.github/workflows/mcp_conformance.yml +++ b/.github/workflows/mcp_conformance.yml @@ -67,7 +67,7 @@ jobs: MCP_CONFORMANCE_TOKEN: pull-only run: >- docker compose -f tests/conformance/docker-compose.yml - pull redis control-plane nginx + pull redis fixture-proxy control-plane nginx - name: Start the fixture and control plane env: @@ -86,20 +86,9 @@ jobs: id: runner run: tests/conformance/run-conformance.sh - - name: Print conformance failures and warnings + - name: Report conformance baseline diff if: always() - run: | - if [ ! -d conformance-results ]; then - echo "No conformance results were produced." - exit 0 - fi - find conformance-results -type f -name checks.json -print0 | - while IFS= read -r -d '' checks_file; do - jq --arg file "${checks_file}" \ - '[.[] | select(.status == "FAILURE" or .status == "WARNING")] | - if length > 0 then {file: $file, findings: .} else empty end' \ - "${checks_file}" - done + run: tests/conformance/report-baseline-diff.sh - name: Print live stack logs if: always() @@ -109,6 +98,15 @@ jobs: docker compose -f tests/conformance/docker-compose.yml logs --no-color || true + - name: Print official fixture log + if: always() + run: | + if [ -f conformance-logs/reference-server.log ]; then + sed -n '1,240p' conformance-logs/reference-server.log + else + echo "No official fixture log was produced." + fi + - name: Stop the live stack if: always() env: @@ -119,4 +117,8 @@ jobs: if: always() env: RUNNER_STATUS: ${{ steps.runner.outputs.status }} - run: test "${RUNNER_STATUS}" = "0" + run: | + if [ "${RUNNER_STATUS}" != "0" ]; then + echo "::error title=Conformance baseline mismatch::Actual findings differ from the expected baseline. See the baseline diff step and job summary." + exit 1 + fi diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 5f6bd6e..65d7644 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -38,16 +38,24 @@ These run in `cargo nextest run` with no Docker dependencies. `.github/workflows/mcp_conformance.yml` runs the pinned official conformance suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is -official runner → nginx → published `latest` dataplane → official fixture, -with the published `latest` control plane registering and publishing the -fixture through Redis. The control plane uses ephemeral SQLite, so PostgreSQL -is unnecessary. The harness lives in `tests/conformance/`. +official runner → nginx → checked-out dataplane → fixture proxy → official +fixture, with the published `latest` control plane registering and publishing +the fixture through Redis. The backend-only proxy rewrites `Host` to +`localhost:3000`, which the official fixture's DNS-rebinding protection +requires, while leaving dataplane header protections unchanged. The control +plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives +in `tests/conformance/`. Because this conformance CLI cannot set a bearer header, nginx adds an ephemeral control-plane token when one is absent; there is no auth proxy or repository-owned JavaScript. A route probe prevents control-plane fallback. -Counts appear directly in the Actions log, and `expected-failures.yml` guards -the current baseline. The job does not retain a separate conformance artifact. +Counts and the official fixture log appear directly in the Actions log, and +`expected-failures.yml` guards the current baseline. The job does not retain a +separate conformance artifact. `upstream-fixture-failures.yml` records the +pinned fixture's seven scored failures and one warning; its other 47 failures +are extension or pending scenarios and are already unscored. CI prints the +exact actual-versus-baseline diff, adds annotations for unexpected and stale +entries, and writes the same comparison to the job summary. ## Full-Stack Integration Harness diff --git a/tests/conformance/docker-compose.yml b/tests/conformance/docker-compose.yml index f941f3c..21e978d 100644 --- a/tests/conformance/docker-compose.yml +++ b/tests/conformance/docker-compose.yml @@ -11,13 +11,34 @@ services: timeout: 2s retries: 30 + fixture-proxy: + image: nginx:1.30.4-alpine3.24 + networks: [contextforge] + extra_hosts: + - host.docker.internal:host-gateway + volumes: + - ./fixture-proxy.conf:/etc/nginx/conf.d/default.conf:ro + healthcheck: + test: + - CMD-SHELL + - >- + curl --fail --silent --show-error --output /dev/null + --request POST + --header 'Content-Type: application/json' + --header 'Accept: application/json, text/event-stream' + --header 'MCP-Protocol-Version: 2026-07-28' + --header 'MCP-Method: server/discover' + --data '{"jsonrpc":"2.0","id":"health","method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{}}}}' + http://127.0.0.1/mcp + interval: 1s + timeout: 5s + retries: 60 + control-plane: image: ${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest} ports: - "127.0.0.1:4444:4444" networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway environment: HOST: 0.0.0.0 PORT: "4444" @@ -62,8 +83,6 @@ services: data-plane: image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest} networks: [contextforge] - extra_hosts: - - host.docker.internal:host-gateway environment: CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis diff --git a/tests/conformance/expected-failures.yml b/tests/conformance/expected-failures.yml index 4f5007b..22fa69e 100644 --- a/tests/conformance/expected-failures.yml +++ b/tests/conformance/expected-failures.yml @@ -7,6 +7,8 @@ server: - server-stateless:sep-2575-server-no-log-without-loglevel - completion-complete:completion-complete - tools-list:tools-list + # Dataplane-owned on main. The fixture passes these behavior checks itself; + # its separate wire-schema failures live in upstream-fixture-failures.yml. - tools-call-simple-text:tools-call-simple-text - tools-call-image:tools-call-image - tools-call-audio:tools-call-audio diff --git a/tests/conformance/fixture-proxy.conf b/tests/conformance/fixture-proxy.conf new file mode 100644 index 0000000..7fac91b --- /dev/null +++ b/tests/conformance/fixture-proxy.conf @@ -0,0 +1,18 @@ +server { + listen 80 default_server; + server_name _; + + location / { + proxy_pass http://host.docker.internal:3000; + proxy_http_version 1.1; + proxy_set_header Host localhost:3000; + proxy_set_header Connection ""; + proxy_request_buffering on; + proxy_buffering off; + proxy_cache off; + proxy_connect_timeout 30s; + proxy_send_timeout 1h; + proxy_read_timeout 1h; + add_header X-Accel-Buffering "no" always; + } +} diff --git a/tests/conformance/register-fixture.sh b/tests/conformance/register-fixture.sh index ffae024..2919276 100755 --- a/tests/conformance/register-fixture.sh +++ b/tests/conformance/register-fixture.sh @@ -36,11 +36,11 @@ api_request() { gateway="$(api_request POST /gateways '{ "name": "_", - "url": "http://host.docker.internal:3000/mcp", + "url": "http://fixture-proxy/mcp", "transport": "STREAMABLEHTTP", "authType": "authheaders", "authHeaders": [{"key": "Host", "value": "localhost:3000"}], - "description": "Official MCP alpha.11 conformance fixture" + "description": "Official MCP alpha.11 conformance fixture through the test-only Host proxy" }')" gateway_id="$(jq --exit-status --raw-output '.id' <<< "${gateway}")" diff --git a/tests/conformance/report-baseline-diff.sh b/tests/conformance/report-baseline-diff.sh new file mode 100755 index 0000000..0656deb --- /dev/null +++ b/tests/conformance/report-baseline-diff.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +results_dir="${1:-${repo_root}/conformance-results}" +baseline_file="${2:-${script_dir}/expected-failures.yml}" +upstream_file="${3:-${script_dir}/upstream-fixture-failures.yml}" +suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" +spec_version="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" +requirements_file="${suite_dir}/requirements/${spec_version}.yaml" + +for command in awk cut find grep jq sed sort wc; do + if ! command -v "${command}" > /dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done + +for required_file in "${baseline_file}" "${upstream_file}" "${requirements_file}"; do + if [ ! -f "${required_file}" ]; then + echo "Required conformance file not found: ${required_file}" >&2 + exit 1 + fi +done + +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-diff.XXXXXX")" +actual_findings="${state_dir}/actual-findings.tsv" +actual_keys="${state_dir}/actual-keys.txt" +baseline_entries="${state_dir}/baseline-entries.txt" +upstream_entries="${state_dir}/upstream-entries.txt" +scored_scenarios="${state_dir}/scored-scenarios.txt" +unexpected_entries="${state_dir}/unexpected-entries.txt" +stale_entries="${state_dir}/stale-entries.txt" +upstream_matches="${state_dir}/upstream-matches.txt" + +cleanup() { + rm -f -- \ + "${actual_findings}" \ + "${actual_keys}" \ + "${baseline_entries}" \ + "${upstream_entries}" \ + "${scored_scenarios}" \ + "${unexpected_entries}" \ + "${stale_entries}" \ + "${upstream_matches}" + rmdir -- "${state_dir}" +} +trap cleanup EXIT INT TERM + +read_baseline() { + awk ' + /^[[:space:]]*-[[:space:]]+/ { + line = $0 + sub(/^[[:space:]]*-[[:space:]]+/, "", line) + sub(/[[:space:]]+#.*$/, "", line) + print line + } + ' "$1" | LC_ALL=C sort -u +} + +awk ' + /^server:$/ { in_server = 1; next } + in_server && /^[^[:space:]]/ { exit } + in_server && /^[[:space:]]*-[[:space:]]+/ { + line = $0 + sub(/^[[:space:]]*-[[:space:]]+/, "", line) + print line + } +' "${requirements_file}" | LC_ALL=C sort -u > "${scored_scenarios}" + +read_baseline "${baseline_file}" > "${baseline_entries}" +read_baseline "${upstream_file}" > "${upstream_entries}" + +if [ ! -d "${results_dir}" ]; then + echo "::warning title=Conformance results missing::No results directory: ${results_dir}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '## Conformance baseline diff\n\nNo conformance results were produced.\n' \ + >> "${GITHUB_STEP_SUMMARY}" + fi + exit 0 +fi + +: > "${actual_findings}" +while IFS= read -r -d '' checks_file; do + case "${checks_file}" in + */checks.json) ;; + *) continue ;; + esac + + result_name="$(basename -- "$(dirname -- "${checks_file}")")" + if [[ ! "${result_name}" =~ ^server-(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z$ ]]; then + echo "Skipping unrecognized result directory: ${result_name}" >&2 + continue + fi + scenario="${BASH_REMATCH[1]}" + + if ! grep --fixed-strings --line-regexp --quiet "${scenario}" "${scored_scenarios}"; then + continue + fi + + jq --raw-output --arg scenario "${scenario}" ' + .[] | + select(.status == "FAILURE" or .status == "WARNING") | + [($scenario + ":" + .id), .status, (.errorMessage // "")] | + @tsv + ' "${checks_file}" >> "${actual_findings}" +done < <(find "${results_dir}" -type f -name checks.json -print0) + +LC_ALL=C sort -u -o "${actual_findings}" "${actual_findings}" +cut -f 1 "${actual_findings}" | LC_ALL=C sort -u > "${actual_keys}" + +awk ' + NR == FNR { expected[$1] = 1; next } + { + scenario = $1 + sub(/:.*/, "", scenario) + if (!(($1 in expected) || (scenario in expected))) print $1 + } +' "${baseline_entries}" "${actual_keys}" > "${unexpected_entries}" + +awk ' + NR == FNR { + actual[$1] = 1 + scenario = $1 + sub(/:.*/, "", scenario) + actual_scenario[scenario] = 1 + next + } + !(($1 in actual) || ($1 in actual_scenario)) { print $1 } +' "${actual_keys}" "${baseline_entries}" > "${stale_entries}" + +awk ' + NR == FNR { upstream[$1] = 1; next } + { + scenario = $1 + sub(/:.*/, "", scenario) + if (($1 in upstream) || (scenario in upstream)) print $1 + } +' "${upstream_entries}" "${actual_keys}" > "${upstream_matches}" + +line_count() { + wc -l < "$1" | tr -d '[:space:]' +} + +actual_count="$(line_count "${actual_keys}")" +baseline_count="$(line_count "${baseline_entries}")" +unexpected_count="$(line_count "${unexpected_entries}")" +stale_count="$(line_count "${stale_entries}")" +upstream_count="$(line_count "${upstream_matches}")" + +echo "Conformance baseline diff (${spec_version})" +echo " Actual scored findings: ${actual_count}" +echo " Expected baseline entries: ${baseline_count}" +echo " Matched pinned-fixture findings: ${upstream_count}" +echo " Unexpected findings: ${unexpected_count}" +echo " Stale baseline entries: ${stale_count}" + +if [ "${unexpected_count}" -gt 0 ]; then + echo + echo "Unexpected findings (actual but not in baseline):" + while IFS= read -r key; do + detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")" + echo " - ${key} — ${detail}" + echo "::error title=Unexpected conformance finding::${key}" + done < "${unexpected_entries}" +fi + +if [ "${stale_count}" -gt 0 ]; then + echo + echo "Stale baseline entries (expected but now passing):" + while IFS= read -r key; do + echo " - ${key}" + echo "::error title=Stale conformance baseline::${key} is now passing" + done < "${stale_entries}" +fi + +if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then + echo "Actual scored findings match the expected baseline." +fi + +if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "## Conformance baseline diff" + echo + echo "| Category | Count |" + echo "| --- | ---: |" + echo "| Actual scored findings | ${actual_count} |" + echo "| Expected baseline entries | ${baseline_count} |" + echo "| Matched pinned-fixture findings | ${upstream_count} |" + echo "| Unexpected findings | ${unexpected_count} |" + echo "| Stale baseline entries | ${stale_count} |" + echo + echo "The pinned alpha.11 fixture has 7 scored failures and 1 warning recorded in \`upstream-fixture-failures.yml\`; its other 47 failures are extension or pending scenarios and are unscored." + + if [ "${unexpected_count}" -gt 0 ]; then + echo + echo "### Unexpected findings" + while IFS= read -r key; do + detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")" + echo "- \`${key}\` — ${detail}" + done < "${unexpected_entries}" + fi + + if [ "${stale_count}" -gt 0 ]; then + echo + echo "### Stale baseline entries" + while IFS= read -r key; do + echo "- \`${key}\`" + done < "${stale_entries}" + fi + + if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then + echo + echo "Actual scored findings match the expected baseline." + fi + } >> "${GITHUB_STEP_SUMMARY}" +fi diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh index 86bf25d..77c5cbb 100755 --- a/tests/conformance/run-local.sh +++ b/tests/conformance/run-local.sh @@ -61,6 +61,10 @@ cleanup() { echo "Conformance run failed; printing live stack logs." >&2 MCP_CONFORMANCE_TOKEN=diagnostics-only \ docker compose -f "${compose_file}" logs --no-color || true + if [ -f "${repo_root}/conformance-logs/reference-server.log" ]; then + echo "Official fixture log:" >&2 + sed -n '1,240p' "${repo_root}/conformance-logs/reference-server.log" >&2 + fi fi MCP_CONFORMANCE_TOKEN="${MCP_CONFORMANCE_TOKEN:-cleanup-only}" \ "${script_dir}/stop-live-stack.sh" || true @@ -71,7 +75,7 @@ cleanup() { trap cleanup EXIT INT TERM MCP_CONFORMANCE_TOKEN=pull-only \ - docker compose -f "${compose_file}" pull redis control-plane nginx + docker compose -f "${compose_file}" pull redis fixture-proxy control-plane nginx echo "Starting the fixture and control plane." MCP_CONFORMANCE_TOKEN=bootstrap-only \ "${script_dir}/start-fixture-and-control-plane.sh" diff --git a/tests/conformance/start-fixture-and-control-plane.sh b/tests/conformance/start-fixture-and-control-plane.sh index a255270..667d8f7 100755 --- a/tests/conformance/start-fixture-and-control-plane.sh +++ b/tests/conformance/start-fixture-and-control-plane.sh @@ -15,7 +15,7 @@ mkdir -p "${log_dir}" "${repo_root}/conformance-results" echo "$!" > "${log_dir}/reference-server.pid" docker compose -f "${script_dir}/docker-compose.yml" \ - up -d --wait redis control-plane + up -d --wait redis fixture-proxy control-plane for _ in $(seq 1 120); do if curl --silent --output /dev/null http://127.0.0.1:3000/mcp; then diff --git a/tests/conformance/upstream-fixture-failures.yml b/tests/conformance/upstream-fixture-failures.yml new file mode 100644 index 0000000..08239bd --- /dev/null +++ b/tests/conformance/upstream-fixture-failures.yml @@ -0,0 +1,14 @@ +# The pinned alpha.11 JavaScript fixture reports 129 passed and 54 failed when +# the official runner targets it directly at MCP 2026-07-28, plus one scored +# warning. Of those failures, 47 belong to extension or pending scenarios and +# are already unscored by the requirements file. These are its seven scored +# failures and one warning. Update this file with MCP_CONFORMANCE_SOURCE_SHA. +server: + - tools-call-simple-text:wire-schema-valid + - tools-call-image:wire-schema-valid + - tools-call-audio:wire-schema-valid + - tools-call-embedded-resource:wire-schema-valid + - tools-call-mixed-content:wire-schema-valid + - tools-call-error:wire-schema-valid + - tools-call-with-progress:wire-schema-valid + - input-required-result-validate-input:sep-2322-validate-input-responses