Route conformance backend through Host proxy

.github/workflows/mcp_conformance.yml

@@ -67,7 +67,7 @@ jobs:
           MCP_CONFORMANCE_TOKEN: pull-only
         run: >-
           docker compose -f tests/conformance/docker-compose.yml
-          pull redis control-plane nginx
+          pull redis fixture-proxy control-plane nginx
 
       - name: Start the fixture and control plane
         env:
@@ -86,20 +86,9 @@ jobs:
         id: runner
         run: tests/conformance/run-conformance.sh
 
-      - name: Print conformance failures and warnings
+      - name: Report conformance baseline diff
         if: always()
-        run: |
-          if [ ! -d conformance-results ]; then
-            echo "No conformance results were produced."
-            exit 0
-          fi
-          find conformance-results -type f -name checks.json -print0 |
-            while IFS= read -r -d '' checks_file; do
-              jq --arg file "${checks_file}" \
-                '[.[] | select(.status == "FAILURE" or .status == "WARNING")] |
-                 if length > 0 then {file: $file, findings: .} else empty end' \
-                "${checks_file}"
-            done
+        run: tests/conformance/report-baseline-diff.sh
 
       - name: Print live stack logs
         if: always()
@@ -109,6 +98,15 @@ jobs:
           docker compose -f tests/conformance/docker-compose.yml
           logs --no-color || true
 
+      - name: Print official fixture log
+        if: always()
+        run: |
+          if [ -f conformance-logs/reference-server.log ]; then
+            sed -n '1,240p' conformance-logs/reference-server.log
+          else
+            echo "No official fixture log was produced."
+          fi
+
       - name: Stop the live stack
         if: always()
         env:
@@ -119,4 +117,8 @@ jobs:
         if: always()
         env:
           RUNNER_STATUS: ${{ steps.runner.outputs.status }}
-        run: test "${RUNNER_STATUS}" = "0"
+        run: |
+          if [ "${RUNNER_STATUS}" != "0" ]; then
+            echo "::error title=Conformance baseline mismatch::Actual findings differ from the expected baseline. See the baseline diff step and job summary."
+            exit 1
+          fi

_context/wiki/testing.md

@@ -38,16 +38,24 @@ These run in `cargo nextest run` with no Docker dependencies.
 
 `.github/workflows/mcp_conformance.yml` runs the pinned official conformance
 suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is
-official runner → nginx → published `latest` dataplane → official fixture,
-with the published `latest` control plane registering and publishing the
-fixture through Redis. The control plane uses ephemeral SQLite, so PostgreSQL
-is unnecessary. The harness lives in `tests/conformance/`.
+official runner → nginx → checked-out dataplane → fixture proxy → official
+fixture, with the published `latest` control plane registering and publishing
+the fixture through Redis. The backend-only proxy rewrites `Host` to
+`localhost:3000`, which the official fixture's DNS-rebinding protection
+requires, while leaving dataplane header protections unchanged. The control
+plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives
+in `tests/conformance/`.
 
 Because this conformance CLI cannot set a bearer header, nginx adds an
 ephemeral control-plane token when one is absent; there is no auth proxy or
 repository-owned JavaScript. A route probe prevents control-plane fallback.
-Counts appear directly in the Actions log, and `expected-failures.yml` guards
-the current baseline. The job does not retain a separate conformance artifact.
+Counts and the official fixture log appear directly in the Actions log, and
+`expected-failures.yml` guards the current baseline. The job does not retain a
+separate conformance artifact. `upstream-fixture-failures.yml` records the
+pinned fixture's seven scored failures and one warning; its other 47 failures
+are extension or pending scenarios and are already unscored. CI prints the
+exact actual-versus-baseline diff, adds annotations for unexpected and stale
+entries, and writes the same comparison to the job summary.
 
 ## Full-Stack Integration Harness
 

tests/conformance/docker-compose.yml

@@ -11,13 +11,34 @@ services:
       timeout: 2s
       retries: 30
 
+  fixture-proxy:
+    image: nginx:1.30.4-alpine3.24
+    networks: [contextforge]
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    volumes:
+      - ./fixture-proxy.conf:/etc/nginx/conf.d/default.conf:ro
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - >-
+          curl --fail --silent --show-error --output /dev/null
+          --request POST
+          --header 'Content-Type: application/json'
+          --header 'Accept: application/json, text/event-stream'
+          --header 'MCP-Protocol-Version: 2026-07-28'
+          --header 'MCP-Method: server/discover'
+          --data '{"jsonrpc":"2.0","id":"health","method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientCapabilities":{}}}}'
+          http://127.0.0.1/mcp
+      interval: 1s
+      timeout: 5s
+      retries: 60
+
   control-plane:
     image: ${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest}
     ports:
       - "127.0.0.1:4444:4444"
     networks: [contextforge]
-    extra_hosts:
-      - host.docker.internal:host-gateway
     environment:
       HOST: 0.0.0.0
       PORT: "4444"
@@ -62,8 +83,6 @@ services:
   data-plane:
     image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest}
     networks: [contextforge]
-    extra_hosts:
-      - host.docker.internal:host-gateway
     environment:
       CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445
       CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis

tests/conformance/expected-failures.yml

@@ -7,6 +7,8 @@ server:
   - server-stateless:sep-2575-server-no-log-without-loglevel
   - completion-complete:completion-complete
   - tools-list:tools-list
+  # Dataplane-owned on main. The fixture passes these behavior checks itself;
+  # its separate wire-schema failures live in upstream-fixture-failures.yml.
   - tools-call-simple-text:tools-call-simple-text
   - tools-call-image:tools-call-image
   - tools-call-audio:tools-call-audio

tests/conformance/fixture-proxy.conf

@@ -0,0 +1,18 @@
+server {
+    listen 80 default_server;
+    server_name _;
+
+    location / {
+        proxy_pass http://host.docker.internal:3000;
+        proxy_http_version 1.1;
+        proxy_set_header Host localhost:3000;
+        proxy_set_header Connection "";
+        proxy_request_buffering on;
+        proxy_buffering off;
+        proxy_cache off;
+        proxy_connect_timeout 30s;
+        proxy_send_timeout 1h;
+        proxy_read_timeout 1h;
+        add_header X-Accel-Buffering "no" always;
+    }
+}

tests/conformance/register-fixture.sh

@@ -36,11 +36,11 @@ api_request() {
 
 gateway="$(api_request POST /gateways '{
   "name": "_",
-  "url": "http://host.docker.internal:3000/mcp",
+  "url": "http://fixture-proxy/mcp",
   "transport": "STREAMABLEHTTP",
   "authType": "authheaders",
   "authHeaders": [{"key": "Host", "value": "localhost:3000"}],
-  "description": "Official MCP alpha.11 conformance fixture"
+  "description": "Official MCP alpha.11 conformance fixture through the test-only Host proxy"
 }')"
 gateway_id="$(jq --exit-status --raw-output '.id' <<< "${gateway}")"
 

tests/conformance/report-baseline-diff.sh

@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+repo_root="$(cd -- "${script_dir}/../.." && pwd)"
+results_dir="${1:-${repo_root}/conformance-results}"
+baseline_file="${2:-${script_dir}/expected-failures.yml}"
+upstream_file="${3:-${script_dir}/upstream-fixture-failures.yml}"
+suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}"
+spec_version="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}"
+requirements_file="${suite_dir}/requirements/${spec_version}.yaml"
+
+for command in awk cut find grep jq sed sort wc; do
+  if ! command -v "${command}" > /dev/null 2>&1; then
+    echo "Required command not found: ${command}" >&2
+    exit 1
+  fi
+done
+
+for required_file in "${baseline_file}" "${upstream_file}" "${requirements_file}"; do
+  if [ ! -f "${required_file}" ]; then
+    echo "Required conformance file not found: ${required_file}" >&2
+    exit 1
+  fi
+done
+
+state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-baseline-diff.XXXXXX")"
+actual_findings="${state_dir}/actual-findings.tsv"
+actual_keys="${state_dir}/actual-keys.txt"
+baseline_entries="${state_dir}/baseline-entries.txt"
+upstream_entries="${state_dir}/upstream-entries.txt"
+scored_scenarios="${state_dir}/scored-scenarios.txt"
+unexpected_entries="${state_dir}/unexpected-entries.txt"
+stale_entries="${state_dir}/stale-entries.txt"
+upstream_matches="${state_dir}/upstream-matches.txt"
+
+cleanup() {
+  rm -f -- \
+    "${actual_findings}" \
+    "${actual_keys}" \
+    "${baseline_entries}" \
+    "${upstream_entries}" \
+    "${scored_scenarios}" \
+    "${unexpected_entries}" \
+    "${stale_entries}" \
+    "${upstream_matches}"
+  rmdir -- "${state_dir}"
+}
+trap cleanup EXIT INT TERM
+
+read_baseline() {
+  awk '
+    /^[[:space:]]*-[[:space:]]+/ {
+      line = $0
+      sub(/^[[:space:]]*-[[:space:]]+/, "", line)
+      sub(/[[:space:]]+#.*$/, "", line)
+      print line
+    }
+  ' "$1" | LC_ALL=C sort -u
+}
+
+awk '
+  /^server:$/ { in_server = 1; next }
+  in_server && /^[^[:space:]]/ { exit }
+  in_server && /^[[:space:]]*-[[:space:]]+/ {
+    line = $0
+    sub(/^[[:space:]]*-[[:space:]]+/, "", line)
+    print line
+  }
+' "${requirements_file}" | LC_ALL=C sort -u > "${scored_scenarios}"
+
+read_baseline "${baseline_file}" > "${baseline_entries}"
+read_baseline "${upstream_file}" > "${upstream_entries}"
+
+if [ ! -d "${results_dir}" ]; then
+  echo "::warning title=Conformance results missing::No results directory: ${results_dir}"
+  if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+    printf '## Conformance baseline diff\n\nNo conformance results were produced.\n' \
+      >> "${GITHUB_STEP_SUMMARY}"
+  fi
+  exit 0
+fi
+
+: > "${actual_findings}"
+while IFS= read -r -d '' checks_file; do
+  case "${checks_file}" in
+    */checks.json) ;;
+    *) continue ;;
+  esac
+
+  result_name="$(basename -- "$(dirname -- "${checks_file}")")"
+  if [[ ! "${result_name}" =~ ^server-(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]{3}Z$ ]]; then
+    echo "Skipping unrecognized result directory: ${result_name}" >&2
+    continue
+  fi
+  scenario="${BASH_REMATCH[1]}"
+
+  if ! grep --fixed-strings --line-regexp --quiet "${scenario}" "${scored_scenarios}"; then
+    continue
+  fi
+
+  jq --raw-output --arg scenario "${scenario}" '
+    .[] |
+    select(.status == "FAILURE" or .status == "WARNING") |
+    [($scenario + ":" + .id), .status, (.errorMessage // "")] |
+    @tsv
+  ' "${checks_file}" >> "${actual_findings}"
+done < <(find "${results_dir}" -type f -name checks.json -print0)
+
+LC_ALL=C sort -u -o "${actual_findings}" "${actual_findings}"
+cut -f 1 "${actual_findings}" | LC_ALL=C sort -u > "${actual_keys}"
+
+awk '
+  NR == FNR { expected[$1] = 1; next }
+  {
+    scenario = $1
+    sub(/:.*/, "", scenario)
+    if (!(($1 in expected) || (scenario in expected))) print $1
+  }
+' "${baseline_entries}" "${actual_keys}" > "${unexpected_entries}"
+
+awk '
+  NR == FNR {
+    actual[$1] = 1
+    scenario = $1
+    sub(/:.*/, "", scenario)
+    actual_scenario[scenario] = 1
+    next
+  }
+  !(($1 in actual) || ($1 in actual_scenario)) { print $1 }
+' "${actual_keys}" "${baseline_entries}" > "${stale_entries}"
+
+awk '
+  NR == FNR { upstream[$1] = 1; next }
+  {
+    scenario = $1
+    sub(/:.*/, "", scenario)
+    if (($1 in upstream) || (scenario in upstream)) print $1
+  }
+' "${upstream_entries}" "${actual_keys}" > "${upstream_matches}"
+
+line_count() {
+  wc -l < "$1" | tr -d '[:space:]'
+}
+
+actual_count="$(line_count "${actual_keys}")"
+baseline_count="$(line_count "${baseline_entries}")"
+unexpected_count="$(line_count "${unexpected_entries}")"
+stale_count="$(line_count "${stale_entries}")"
+upstream_count="$(line_count "${upstream_matches}")"
+
+echo "Conformance baseline diff (${spec_version})"
+echo "  Actual scored findings: ${actual_count}"
+echo "  Expected baseline entries: ${baseline_count}"
+echo "  Matched pinned-fixture findings: ${upstream_count}"
+echo "  Unexpected findings: ${unexpected_count}"
+echo "  Stale baseline entries: ${stale_count}"
+
+if [ "${unexpected_count}" -gt 0 ]; then
+  echo
+  echo "Unexpected findings (actual but not in baseline):"
+  while IFS= read -r key; do
+    detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")"
+    echo "  - ${key} — ${detail}"
+    echo "::error title=Unexpected conformance finding::${key}"
+  done < "${unexpected_entries}"
+fi
+
+if [ "${stale_count}" -gt 0 ]; then
+  echo
+  echo "Stale baseline entries (expected but now passing):"
+  while IFS= read -r key; do
+    echo "  - ${key}"
+    echo "::error title=Stale conformance baseline::${key} is now passing"
+  done < "${stale_entries}"
+fi
+
+if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then
+  echo "Actual scored findings match the expected baseline."
+fi
+
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+  {
+    echo "## Conformance baseline diff"
+    echo
+    echo "| Category | Count |"
+    echo "| --- | ---: |"
+    echo "| Actual scored findings | ${actual_count} |"
+    echo "| Expected baseline entries | ${baseline_count} |"
+    echo "| Matched pinned-fixture findings | ${upstream_count} |"
+    echo "| Unexpected findings | ${unexpected_count} |"
+    echo "| Stale baseline entries | ${stale_count} |"
+    echo
+    echo "The pinned alpha.11 fixture has 7 scored failures and 1 warning recorded in \`upstream-fixture-failures.yml\`; its other 47 failures are extension or pending scenarios and are unscored."
+
+    if [ "${unexpected_count}" -gt 0 ]; then
+      echo
+      echo "### Unexpected findings"
+      while IFS= read -r key; do
+        detail="$(awk -F '\t' -v key="${key}" '$1 == key { print $2 ": " $3; exit }' "${actual_findings}")"
+        echo "- \`${key}\` — ${detail}"
+      done < "${unexpected_entries}"
+    fi
+
+    if [ "${stale_count}" -gt 0 ]; then
+      echo
+      echo "### Stale baseline entries"
+      while IFS= read -r key; do
+        echo "- \`${key}\`"
+      done < "${stale_entries}"
+    fi
+
+    if [ "${unexpected_count}" -eq 0 ] && [ "${stale_count}" -eq 0 ]; then
+      echo
+      echo "Actual scored findings match the expected baseline."
+    fi
+  } >> "${GITHUB_STEP_SUMMARY}"
+fi