test(auth): add a SAML lane to the mock-IdP stack to reproduce #263

.github/workflows/mock-idp-e2e.yml

@@ -244,6 +244,224 @@ jobs:
         if: always()
         run: docker compose -f compose.mock-idp.yml down -v
 
+  # DIAGNOSTIC job for issue #263 (Workbench behind SAML on a separate
+  # hostname from Connect). This job's expected outcome is RED -- its
+  # purpose is to reproduce a suspected defect in `_authenticate_workbench`
+  # so the fix (to src/vip/auth.py) can be written against an observed
+  # failure rather than an inference. See
+  # thoughts/shared/plans/2026-08-17-issue-263-saml-diagnostic-plan.md.
+  #
+  # It therefore:
+  #   - runs only on workflow_dispatch (never push/PR/schedule), so it can
+  #     never block a merge or a nightly;
+  #   - sets `continue-on-error: true` at the job level, so even a failing
+  #     run reports green to anything that depends on this workflow;
+  #   - is deliberately NOT in the `status` job's `needs:` below, so it can
+  #     never affect that aggregation either.
+  # Once the auth.py fix lands and this job goes green on its own merits,
+  # promote it: drop `continue-on-error`, fold it into (or replace)
+  # `mock-idp-e2e` above, and add it to `status`'s `needs:`.
+  mock-idp-e2e-saml-diagnostic:
+    if: github.event_name == 'workflow_dispatch'
+    name: Mock-IdP E2E SAML diagnostic (issue #263, expected red)
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      # Same as the headless-auth job's hostname step, plus the SAML lane's
+      # own hostname (docker/workbench/entrypoint-saml.sh,
+      # compose.mock-idp.yml's `workbench-saml` service).
+      - name: Add vip.test hostnames
+        run: |
+          echo "127.0.0.1 keycloak.vip.test connect.vip.test workbench.vip.test workbench-saml.vip.test" | sudo tee -a /etc/hosts
+
+      # `--profile saml` additionally builds and starts `workbench-saml`
+      # (see compose.mock-idp.yml); the OIDC `workbench` service and
+      # everything else in the stack come up exactly as they do in the
+      # headless-auth job above.
+      - name: Start mock-IdP stack with SAML lane (Keycloak + Connect + Workbench + Workbench-SAML)
+        env:
+          RSC_LICENSE: ${{ secrets.CONNECT_LICENSE }}
+          RSW_LICENSE: ${{ secrets.WORKBENCH_LICENSE }}
+        run: docker compose -f compose.mock-idp.yml --profile saml up -d --build --wait
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
+
+      - name: Build VIP runner image
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
+        with:
+          context: .
+          load: true
+          tags: vip-mock-idp-runner:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Create output directory for test results
+        run: mkdir -m 777 -p mock-idp-out
+
+      # Two runs, mirroring Task 4 of the diagnostic plan: (a) Workbench-only
+      # against the SAML lane, (b) both products on separate hostnames --
+      # the actual #263 topology. `set -o pipefail` is required here: the
+      # default GHA bash shell runs `bash -e {0}` with no pipefail, so
+      # `docker run ... | tee file` would otherwise report this step as
+      # successful even when `vip verify` inside the container fails.
+      #
+      # Step (b) below carries `if: always()` so it still runs -- and its
+      # log still gets captured -- even when this step fails, which is the
+      # "continuing past failure so we get both logs" requirement. Neither
+      # step swallows its own exit code (no `|| true`), so a real failure
+      # here still fails the step, which is what lets "Surface stack logs on
+      # failure" below fire via the normal `if: failure()` idiom; the job's
+      # `continue-on-error: true` above is what keeps that from blocking
+      # anything downstream.
+      #
+      # NOTE: --interactive-auth is NOT covered by this diagnostic, for the
+      # same reason it is not exercised by the headless-auth job above (see
+      # that job's comment) -- it opens a headed browser and blocks on a
+      # human. Issue #263 was reported under --interactive-auth; this probes
+      # with --headless-auth because both modes share `_authenticate_workbench`
+      # in auth.py, which is where the leading hypothesis sits -- but a
+      # defect specific to the interactive-only prompt path would not show
+      # up here.
+      - name: Run SAML diagnostic (Workbench-only)
+        run: |
+          set -o pipefail
+          docker run --rm --network host --user root \
+            -v vip-mock-idp_mock-idp-certs:/certs:ro \
+            -v "${{ github.workspace }}/mock-idp-out:/app/out" \
+            -e VIP_TEST_USERNAME=vip_test \
+            -e VIP_TEST_PASSWORD=vip_test_password \
+            --entrypoint sh \
+            vip-mock-idp-runner:latest \
+            -c '
+              cp /certs/ca.crt /usr/local/share/ca-certificates/vip-mock-idp-ca.crt &&
+              update-ca-certificates &&
+              apt-get update -qq && apt-get install -qq -y libnss3-tools > /dev/null &&
+              mkdir -p "$HOME/.pki/nssdb" &&
+              certutil -d "sql:$HOME/.pki/nssdb" -N --empty-password &&
+              certutil -d "sql:$HOME/.pki/nssdb" -A -t "C,," -n vip-mock-idp-ca -i /certs/ca.crt &&
+              export VIP_TEST_TOTP_SECRET="$(cat /certs/totp-secret.b32)" &&
+              exec uv run vip verify \
+                --workbench-url https://workbench-saml.vip.test:8788 \
+                --ca-bundle /certs/ca.crt \
+                --headless-auth --idp keycloak --provider saml \
+                --categories workbench \
+                --filter test_workbench_login \
+                --report out/results-saml-workbench-only.json \
+                -- --junitxml=out/mock-idp-results-saml-workbench-only.xml --vip-verbose
+            ' 2>&1 | tee mock-idp-out/saml-workbench-only.log
+
+          # TEMPORARY: a genuine Workbench auth failure surfaces as
+          # pytest.skip, so `vip verify` exits 0 and this step reports a
+          # false green even though the login test we care about never ran
+          # ("1 passed, 2 skipped"). The durable fix is #596 (a real vip
+          # verify exit-code contract); until that lands, assert directly
+          # that the login test(s) for the product URL(s) passed above
+          # actually executed. Checking named tests rather than the
+          # testsuite's skipped= count matters: prerequisites/
+          # test_expected_failure always skips unless
+          # VIP_ENABLE_EXPECTED_FAILURE_DEMO is set, so a count-based guard
+          # would fail every run regardless of #263. Deliberately duplicated
+          # in both steps so the whole block lifts out cleanly once #596
+          # ships.
+          python3 - mock-idp-out/mock-idp-results-saml-workbench-only.xml test_workbench_login <<'PY'
+          import sys, xml.etree.ElementTree as ET
+
+          xml_path, *required = sys.argv[1:]
+          cases = {tc.get("name", ""): tc for tc in ET.parse(xml_path).iter("testcase")}
+          problems = []
+          for want in required:
+              matched = [n for n in cases if want in n]
+              if not matched:
+                  problems.append(f"{want}: never collected")
+              problems += [f"{n}: skipped" for n in matched if cases[n].find("skipped") is not None]
+          if problems:
+              print("::error::The SAML workbench-only run did not execute its login test(s) -- " + "; ".join(problems) + " (see #596)")
+              sys.exit(1)
+          PY
+
+      - name: Run SAML diagnostic (Connect + Workbench, the #263 topology)
+        if: always()
+        run: |
+          set -o pipefail
+          docker run --rm --network host --user root \
+            -v vip-mock-idp_mock-idp-certs:/certs:ro \
+            -v "${{ github.workspace }}/mock-idp-out:/app/out" \
+            -e VIP_TEST_USERNAME=vip_test \
+            -e VIP_TEST_PASSWORD=vip_test_password \
+            --entrypoint sh \
+            vip-mock-idp-runner:latest \
+            -c '
+              cp /certs/ca.crt /usr/local/share/ca-certificates/vip-mock-idp-ca.crt &&
+              update-ca-certificates &&
+              apt-get update -qq && apt-get install -qq -y libnss3-tools > /dev/null &&
+              mkdir -p "$HOME/.pki/nssdb" &&
+              certutil -d "sql:$HOME/.pki/nssdb" -N --empty-password &&
+              certutil -d "sql:$HOME/.pki/nssdb" -A -t "C,," -n vip-mock-idp-ca -i /certs/ca.crt &&
+              export VIP_TEST_TOTP_SECRET="$(cat /certs/totp-secret.b32)" &&
+              exec uv run vip verify \
+                --connect-url https://connect.vip.test:3939 \
+                --workbench-url https://workbench-saml.vip.test:8788 \
+                --ca-bundle /certs/ca.crt \
+                --headless-auth --idp keycloak --provider saml \
+                --categories "connect or workbench" \
+                --filter "test_connect_login_ui or test_workbench_login" \
+                --report out/results-saml-both.json \
+                -- --junitxml=out/mock-idp-results-saml-both.xml --vip-verbose
+            ' 2>&1 | tee mock-idp-out/saml-both.log
+
+          # TEMPORARY: a genuine Workbench auth failure surfaces as
+          # pytest.skip, so `vip verify` exits 0 and this step reports a
+          # false green even though the login test we care about never ran
+          # ("1 passed, 2 skipped"). The durable fix is #596 (a real vip
+          # verify exit-code contract); until that lands, assert directly
+          # that the login test(s) for the product URL(s) passed above
+          # actually executed. Checking named tests rather than the
+          # testsuite's skipped= count matters: prerequisites/
+          # test_expected_failure always skips unless
+          # VIP_ENABLE_EXPECTED_FAILURE_DEMO is set, so a count-based guard
+          # would fail every run regardless of #263. Deliberately duplicated
+          # in both steps so the whole block lifts out cleanly once #596
+          # ships.
+          python3 - mock-idp-out/mock-idp-results-saml-both.xml test_connect_login_ui test_workbench_login <<'PY'
+          import sys, xml.etree.ElementTree as ET
+
+          xml_path, *required = sys.argv[1:]
+          cases = {tc.get("name", ""): tc for tc in ET.parse(xml_path).iter("testcase")}
+          problems = []
+          for want in required:
+              matched = [n for n in cases if want in n]
+              if not matched:
+                  problems.append(f"{want}: never collected")
+              problems += [f"{n}: skipped" for n in matched if cases[n].find("skipped") is not None]
+          if problems:
+              print("::error::The SAML Connect+Workbench run did not execute its login test(s) -- " + "; ".join(problems) + " (see #596)")
+              sys.exit(1)
+          PY
+
+      # Same placement rationale as the headless-auth job's identical step:
+      # after both E2E attempts, not just after stack startup, so a login-flow
+      # failure -- not just a container-startup failure -- still surfaces logs.
+      - name: Surface stack logs on failure
+        if: failure()
+        run: docker compose -f compose.mock-idp.yml --profile saml logs
+
+      - name: Upload SAML diagnostic results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: mock-idp-e2e-saml-diagnostic-results
+          path: mock-idp-out/
+          if-no-files-found: ignore
+
+      - name: Stop mock-IdP stack
+        if: always()
+        run: docker compose -f compose.mock-idp.yml --profile saml down -v
+
   status:
     name: Mock-IdP E2E Status
     if: always()

compose.mock-idp.yml

@@ -12,14 +12,25 @@
 # /etc/hosts entries pointing all three names at 127.0.0.1 (see
 # .github/workflows/mock-idp-e2e.yml and `just mock-idp-up`).
 #
+# A fourth hostname, workbench-saml.vip.test, is a second Workbench lane
+# configured for SAML instead of OIDC against the same Keycloak realm. It
+# exists to reproduce issue #263 (Workbench behind SAML on a separate
+# hostname from Connect): the OIDC lane above is green daily, which proves
+# the cross-hostname piggyback works over OIDC, so SAML is the one variable
+# it does not cover. SAML is exclusive of PAM/OIDC on a single Workbench
+# instance, so this lane is a separate service rather than a toggle on
+# `workbench`, and it sits behind the `saml` profile so the default
+# `just mock-idp-up` topology (and the OIDC lane's behavior) is unchanged.
+#
 # `cert-init` also generates a random TOTP seed for the realm's test user
 # and renders the realm import file with it (docker/keycloak/gen-totp-secret.sh)
 # rather than baking a fixed secret into the repo. Read it back with
 # `just mock-idp-totp-secret` and export it as VIP_TEST_TOTP_SECRET.
 #
 # Usage:
-#   just mock-idp-up      # generate certs + TOTP seed, start Keycloak + Connect + Workbench
-#   just mock-idp-down    # tear down and remove the generated certs/volumes
+#   just mock-idp-up        # generate certs + TOTP seed, start Keycloak + Connect + Workbench
+#   just mock-idp-saml-up   # same, plus the workbench-saml SAML lane (issue #263)
+#   just mock-idp-down      # tear down and remove the generated certs/volumes
 #
 # License keys (required, same as compose.yml):
 #   RSC_LICENSE, RSW_LICENSE
@@ -133,6 +144,42 @@ services:
       retries: 12
       start_period: 30s
 
+  # Second Workbench lane, configured for SAML instead of OIDC against the
+  # same Keycloak realm (issue #263). Behind the `saml` profile so it never
+  # starts as part of the default `just mock-idp-up` topology -- the `workbench`
+  # service above (the OIDC control) is untouched.
+  workbench-saml:
+    profiles: ["saml"]
+    build:
+      context: ./docker/workbench
+      dockerfile: Dockerfile.saml
+      args:
+        RSW_VERSION: ${RSW_VERSION:-latest}
+    depends_on:
+      cert-init:
+        condition: service_completed_successfully
+      keycloak:
+        condition: service_healthy
+    environment:
+      RSW_LICENSE: ${RSW_LICENSE:-}
+      VIP_TEST_USERNAME: ${VIP_TEST_USERNAME:-vip_test}
+      VIP_TEST_PASSWORD: ${VIP_TEST_PASSWORD:-vip_test_password}
+    volumes:
+      - workbench-saml-home:/home
+      - mock-idp-certs:/certs:ro
+    ports:
+      - "127.0.0.1:8788:8787"
+    networks:
+      vip-mock-idp:
+        aliases:
+          - workbench-saml.vip.test
+    healthcheck:
+      test: ["CMD", "curl", "-fk", "https://localhost:8787"]
+      interval: 10s
+      timeout: 5s
+      retries: 12
+      start_period: 30s
+
 networks:
   vip-mock-idp:
     driver: bridge
@@ -142,3 +189,4 @@ volumes:
   mock-idp-realm-import:
   connect-oidc-data:
   workbench-oidc-home:
+  workbench-saml-home:

docker/keycloak/realm-vip.json

@@ -37,6 +37,37 @@
         "https://workbench.vip.test:8787/*"
       ],
       "webOrigins": ["https://workbench.vip.test:8787"]
+    },
+    {
+      "clientId": "https://workbench-saml.vip.test:8788/saml/metadata",
+      "name": "VIP Workbench SAML (mock-IdP E2E)",
+      "enabled": true,
+      "protocol": "saml",
+      "redirectUris": [
+        "https://workbench-saml.vip.test:8788/saml/acs"
+      ],
+      "attributes": {
+        "saml.assertion.signature": "true",
+        "saml.client.signature": "false",
+        "saml.server.signature": "true",
+        "saml.encrypt": "false",
+        "saml.authnstatement": "true",
+        "saml_name_id_format": "username",
+        "saml_force_name_id_format": "false"
+      },
+      "protocolMappers": [
+        {
+          "name": "username",
+          "protocol": "saml",
+          "protocolMapper": "saml-user-property-mapper",
+          "consentRequired": false,
+          "config": {
+            "attribute.nameformat": "Basic",
+            "user.attribute": "username",
+            "attribute.name": "Username"
+          }
+        }
+      ]
     }
   ],
   "users": [

docker/tls/gen-certs.sh

@@ -15,28 +15,35 @@ set -eu
 #   keycloak.crt/.key      -- leaf cert for keycloak.vip.test
 #   connect.crt/.key       -- leaf cert for connect.vip.test
 #   workbench.crt/.key     -- leaf cert for workbench.vip.test
+#   workbench-saml.crt/.key -- leaf cert for workbench-saml.vip.test (issue #263 SAML lane)
 #
-# Idempotent: skips generation if ca.crt already exists, so re-running
-# `docker compose up` against a warm volume doesn't rotate certs underneath
-# a running Keycloak/Connect/Workbench.
+# Idempotent per-artifact: reuses ca.crt/ca.key if they already exist (so
+# re-running `docker compose up` against a warm volume never rotates the CA
+# underneath a running Keycloak/Connect/Workbench), but still generates any
+# leaf cert that's missing -- e.g. a volume warmed before workbench-saml was
+# added to DOMAINS otherwise skipped the whole script on a warm ca.crt and
+# left `just mock-idp-saml-up` with no workbench-saml.crt/.key to mount.
 
 OUT="${OUT:-/certs}"
-DOMAINS="keycloak connect workbench"
-
-if [ -f "${OUT}/ca.crt" ]; then
-  echo "gen-certs: ${OUT}/ca.crt already exists, skipping generation."
-  exit 0
-fi
+DOMAINS="keycloak connect workbench workbench-saml"
 
 mkdir -p "${OUT}"
 
-echo "gen-certs: generating CA ..."
-openssl req -x509 -newkey rsa:4096 -sha256 -days 30 -nodes \
-  -keyout "${OUT}/ca.key" -out "${OUT}/ca.crt" \
-  -subj "/CN=VIP Mock-IdP E2E CA"
+if [ -f "${OUT}/ca.crt" ]; then
+  echo "gen-certs: ${OUT}/ca.crt already exists, reusing CA."
+else
+  echo "gen-certs: generating CA ..."
+  openssl req -x509 -newkey rsa:4096 -sha256 -days 30 -nodes \
+    -keyout "${OUT}/ca.key" -out "${OUT}/ca.crt" \
+    -subj "/CN=VIP Mock-IdP E2E CA"
+fi
 
 for name in ${DOMAINS}; do
   domain="${name}.vip.test"
+  if [ -f "${OUT}/${name}.crt" ]; then
+    echo "gen-certs: ${OUT}/${name}.crt already exists, skipping."
+    continue
+  fi
   echo "gen-certs: generating leaf cert for ${domain} ..."
   openssl req -newkey rsa:2048 -sha256 -nodes \
     -keyout "${OUT}/${name}.key" -out "${OUT}/${name}.csr" \