diff --git a/README.md b/README.md index 7175e94..464d8f2 100644 --- a/README.md +++ b/README.md @@ -74,12 +74,13 @@ change in one appear in the other, and a conflict when both edit the same note. | --- | --- | | [docs/vision.md](docs/vision.md) | Why krm-foyer exists, who it helps, what it expects from your domain, and what it will not become | | [docs/design.md](docs/design.md) | The contract: routes, access, sessions, upstream responses, streams and release criteria | -| [docs/application-scope.md](docs/application-scope.md) | Why a browser application might be limited beyond RBAC, what that would block, and why krm-foyer starts without it | +| [docs/application-scope.md](docs/application-scope.md) | Why a browser application might be limited beyond RBAC, what that would block, and how to give the browser its own narrow identity in the cluster | | [docs/roadmap.md](docs/roadmap.md) | What exists, what is next, and in what order | | [docs/bounds.md](docs/bounds.md) | Limits, defaults, metrics and the scope of the capacity measurements | | [docs/watches.md](docs/watches.md) | Choosing per-user streams, shared streams or native watches, and configuring sharing | | [docs/bff-choice.md](docs/bff-choice.md) | Whether your application should use a universal BFF like this one, a domain backend, or both | -| [docs/ingress.md](docs/ingress.md) | Terminating TLS itself or behind an ingress, sharing one domain with other services, and the login gate for an ingress | +| [docs/ingress.md](docs/ingress.md) | Terminating TLS itself or behind an ingress, sharing one domain with other services, `/auth/check` for a login gate and a domain backend, and recipes for Gateway API, Traefik, nginx and Vite | +| [docs/room-pass.md](docs/room-pass.md) | Signing an audience in with Room Pass's QR join through krm-foyer | | [docs/frontend.md](docs/frontend.md) | Which pages krm-foyer serves itself, and why it ships no single-page application | | [docs/testing.md](docs/testing.md) | How the tests prove krm-foyer invents neither authentication nor authorization, and how to run the e2e fixture | | [docs/alternatives.md](docs/alternatives.md) | Nearby tools and the goals each meets: other ways to reach the Kubernetes API, and JavaScript clients in place of the browser helper | diff --git a/Taskfile.yml b/Taskfile.yml index 91c09b1..3f95fd8 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -76,11 +76,12 @@ tasks: - test/e2e/cluster/install-traefik.sh e2e-deploy: - desc: "Build the image, deploy krm-foyer into the e2e fixture (as cluster-admin bait), and put the hello example in front of it." + desc: "Build the image, deploy krm-foyer into the e2e fixture (as cluster-admin bait), put the hello example in front of it, and Room Pass's QR login beside it." deps: [image] cmds: - IMAGE={{.IMAGE}}:{{.VERSION}} test/e2e/cluster/deploy-foyer.sh - test/e2e/cluster/front-door.sh + - test/e2e/cluster/room-pass.sh demo: desc: "Start the hello example in a disposable cluster, for a browser on this machine. task e2e-down removes it." @@ -127,6 +128,7 @@ tasks: # The schema and the rendered arguments are tested in cmd/krm-foyer/chart_test.go. - helm lint --strict charts/krm-foyer -f test/e2e/cluster/foyer-values.yaml - helm lint --strict charts/krm-foyer -f test/e2e/cluster/foyer-values.yaml -f test/e2e/cluster/foyer-brief-values.yaml + - helm lint --strict charts/krm-foyer -f test/e2e/cluster/foyer-room-values.yaml fmt: desc: "Format the Go code." diff --git a/cmd/krm-foyer/chart_test.go b/cmd/krm-foyer/chart_test.go index 75435f8..02c7c75 100644 --- a/cmd/krm-foyer/chart_test.go +++ b/cmd/krm-foyer/chart_test.go @@ -221,6 +221,18 @@ func TestChartRendersArgsTheBinaryAccepts(t *testing.T) { t.Fatalf("shared %+v", cfg.login.shared) } }) + t.Run("e2e Room Pass fixture", func(t *testing.T) { + cfg := parseRendered(t, render(t, []string{e2e + "foyer-room-values.yaml"})) + l := cfg.login.auth + if !slices.Equal(l.Scopes, []string{"openid", "email", "profile", "groups", "federated:id"}) || cfg.login.shared != nil { + t.Fatalf("scopes %q, shared %+v", l.Scopes, cfg.login.shared) + } + p := l.Login.AuthorizationParameters + if c := p["connector_id"]; !c.AllowFromRequest || !slices.Equal(c.AllowedValues, []string{"room-pass"}) || len(p) != 1 || + l.Login.SessionClaims.Connector != "/federated_claims/connector_id" { + t.Fatalf("login %+v", l.Login) + } + }) t.Run("every value set", func(t *testing.T) { values := filepath.Join(t.TempDir(), "values.yaml") // Large numbers, which Helm reads from a file as floats, and a fraction. diff --git a/cmd/krm-foyer/main.go b/cmd/krm-foyer/main.go index 62fb85a..f2f0da6 100644 --- a/cmd/krm-foyer/main.go +++ b/cmd/krm-foyer/main.go @@ -353,7 +353,8 @@ func handler(cfg config, logger *slog.Logger, m *metrics.Metrics) (http.Handler, return nil, nil, err } return server.New(server.Config{ - Version: version, Kubernetes: api, Stream: streams, WhoAmI: streams.WhoAmI(), Auth: login.Handler(), Ready: login.Ready, + Version: version, Kubernetes: api, Stream: streams, WhoAmI: streams.WhoAmI(), + Check: login.Check(streams.Identify), Auth: login.Handler(), Ready: login.Ready, }), login.Run, nil } diff --git a/docs/application-scope.md b/docs/application-scope.md index 7edea99..766489e 100644 --- a/docs/application-scope.md +++ b/docs/application-scope.md @@ -94,22 +94,113 @@ which someone has to keep in step with the application. ### A browser identity in Kubernetes -The API server's `AuthenticationConfiguration` can map tokens for the `krm-foyer` -audience to their own identity: a username such as `browser:alice@example.com`, or a -group. RBAC then grants that identity only what the application needs. Kubernetes -enforces the scope itself, and the audit log records it. +**Documented and proved (2026-10-05; client rules tightened 2026-10-06).** This is the first step the decision names, and +it needs no krm-foyer code. Voter's operator asked for it +([implementer feedback](implementer-feedback.md), entry 4): they sign in as a +cluster-admin, and through krm-foyer every script on Voter's origin would hold that. + +The API server's `AuthenticationConfiguration` can give one person two usernames, told +apart by the OIDC client the issuer gave the token to: krm-foyer's tokens one name, the +command line's another. RBAC then grants the browser's name only what the application +needs, and keeps the broad grants on the other. Kubernetes enforces the scope itself, +and the audit log records which name acted. + +The client is the token's **authorized party**: `azp` when the token has one, which +Dex sets when a client asks for another audience with the cross-client scope +`audience:server:client_id:`, and the audience otherwise. The rules are strict, +because a mistake hands out the command line's name: + +- **With `azp`,** it must be a string and one of the token's audiences. +- **Without `azp`,** the audience must be a string or a list of exactly one. A list of + two or more names no client: OpenID Connect requires `azp` with several audiences, and + no rule can say which of them asked. +- **Every client is listed, each with its prefix.** Nothing falls back to a prefix. A + client the list does not name, or a token whose client is in doubt, has no key in + the map; the lookup fails and the API server refuses the token. A validation rule + says so first, with a message. + +So one configuration works for a cluster that accepts krm-foyer's own audience, as the +e2e fixture does, and for one where krm-foyer asks for the cluster's audience, as +Voter's does: -This is the purest form of "Kubernetes decides". Its costs: +```yaml +apiVersion: apiserver.config.k8s.io/v1 +kind: AuthenticationConfiguration +jwt: + - issuer: + url: https://dex.example.com + # Both clients' tokens are accepted. With the cross-client scope krm-foyer's + # carry the audience kubernetes and azp: krm-foyer; the command line's own + # client is kubernetes, and its tokens carry that audience and no azp. + audiences: [kubernetes] + claimMappings: + # The client, as described above, looked up in a map of the clients there are: + # foyer: for the browser, github: for the command line, which + # keeps the cluster-admin binding. CEL has no variables here, so the client is + # spelled out each time it is used. + username: + expression: >- + {'krm-foyer': 'foyer:', 'kubernetes': 'github:'}[(has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : ''))] + + claims.email + # The same split for groups, so that no group binding reaches both. + groups: + expression: >- + has(claims.groups) ? dyn(claims.groups).map(g, + {'krm-foyer': 'foyer:', 'kubernetes': 'github:'}[(has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : ''))] + + g) : [] + claimValidationRules: + # The same client, which must be one in the map. The map lookup above refuses + # any other on its own; this says why. + - expression: >- + (has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : '')) + in ['krm-foyer', 'kubernetes'] + message: "the token's client (azp, or its only audience) must be krm-foyer or kubernetes" + # A username from email is only an identity if the issuer vouched for it. + - expression: "has(claims.email_verified) && type(claims.email_verified) == bool && claims.email_verified" + message: "email_verified must be the boolean true" +``` -- **RBAC twice.** Each grant is written for `alice` and for `browser:alice`, or through a - group for each. -- **Domain logic that matches on identity breaks.** Admission rules and owner fields that - expect `alice` see `browser:alice`. +Then bind the browser's name narrowly: `foyer:simonkoudijs@gmail.com` gets a Role over +the application's resources, and `github:simonkoudijs@gmail.com` keeps `cluster-admin`. + +Which side gets the new name is a choice. Give it to whichever has fewer grants to +rewrite: in Voter's cluster that is the browser, since the existing bindings are for the +command line. The e2e fixture does the reverse, so that its many specs keep their +names: krm-foyer's tokens are `oidc:` and the command line's are +`kubectl:` ([authentication-config.yaml](../test/e2e/cluster/authentication-config.yaml), +with a `kubectl` client in [dex.yaml](../test/e2e/cluster/dex.yaml)). Its spec +([foyer_scope_test.go](../test/e2e/foyer_scope_test.go)) signs alice in both ways, +checks the API server's two names for her, grants the command-line name `list` on +Secrets, and shows krm-foyer refused with the API server's 403 for `oidc:alice`. With +tokens the suite signs itself, it checks every client form above against the real API +server: the browser's and the command line's audiences as strings, as lists of one and +with `azp`, each named and grouped as its client; and an `azp` that is another client, +not an audience, a list, a number, null or empty, two audiences without `azp` in either +order, and another client's audience, each refused with 401, so neither the +command-line name nor a command-line group grant is reachable through them. + +What to know before using it: + +- **A list of audiences needs `azp`.** A token with two audiences and no `azp` is + refused, whatever the order of its audiences, as is an `azp` that is not a string, is + not one of the audiences, or is a client the map does not list. An earlier version + of this recipe compared the client with `krm-foyer` and gave everything else the + command line's prefix: a list audience, or another client's `azp`, got the + command-line name and its groups. Replace any copy of that version. +- **Another client of the same issuer** that has the cluster's audience, as a Dex + `trustedPeers` client can get it, is refused until it is added to the map with a + prefix of its own. +- **RBAC twice.** Each grant the person needs in both places is written for both names, + or through a group for each. +- **Domain logic that matches on identity sees the browser's name.** Admission rules and + owner fields that expect `github:alice` see `foyer:alice`. Attribution through the + [extras](design.md#attribution) is unaffected, since those come from the same claims. - **It needs control of the API server's authentication configuration.** Managed - clusters often do not allow that, so it cannot be the only answer. - -The e2e fixture already configures that authenticator, so it can show this working -without any krm-foyer code. + clusters often do not allow that, so it cannot be the only answer. That is what the + scope list above is for. +- **It scopes by client, not by application.** Two applications behind one krm-foyer + client share the browser's name. Give each its own krm-foyer and client if their + grants should differ. ## How the rest of the design would change @@ -130,5 +221,5 @@ With a scope layer, three things would come back: not fully trusted. - A security review asks what a cross-site-scripting bug in a frontend could reach. -When that happens, document the browser identity first, since it needs no krm-foyer -code, and build the scope list for clusters where that is not possible. +The browser identity is now documented and proved, above. Build the scope list when an +adopter's cluster cannot change its authentication configuration. diff --git a/docs/design.md b/docs/design.md index a6da785..db96a7f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -43,10 +43,11 @@ verifies krm-foyer's certificate. See [both TLS models](ingress.md#decision-2026 Its public URL is configuration: the OIDC redirect URI, same-origin and CSRF checks and return paths use it, never `Host` or `X-Forwarded-*`. An ingress in front must be transparent: no buffering of streams, and no authentication or header rewriting of its own on krm-foyer's -routes. The planned `/auth/check` lets an ingress's external-authentication feature -(`auth_request`, ForwardAuth) act only as a login gate for the application's pages. -Until it exists, the helper's `requireSession()` handles that navigation. A login gate -never decides on `/k8s` or `/stream` traffic. The [ingress decision](ingress.md) explains why, and what would make +routes. `/auth/check` lets an ingress's external-authentication feature +(`auth_request`, ForwardAuth) act as a login gate for the application's pages, and tell a +domain backend on the same origin who the user is, as the API server names them, never +with the token. The helper's `requireSession()` handles the same navigation where no +ingress gate is configured. The check never decides on `/k8s` or `/stream` traffic. The [ingress decision](ingress.md) explains why, and what would make more worth revisiting. krm-foyer has two halves. The **login half** (`/auth/...`) obtains the user's OIDC token @@ -90,7 +91,7 @@ come from krm-stream, not from a reimplementation here. | `/auth/callback` | Validate the callback and establish a session, then `303` to the return path | | `/auth/session` | Return minimal identity/session state and CSRF information, never bearer tokens: `200` with `authenticated`, `issuer`, `subject`, `email`, `displayName`, `groups`, `connector` when configured, `expiresAt`, `csrfToken` and `csrfHeader`, or `401` with `{"authenticated":false}`. See [session claims](#session-claims) | | `/auth/logout` | CSRF-protected POST that clears the session cookie, ends the session's open responses in this process and answers `204`; it revokes no copy of the cookie ([sessions](#sessions)). The caller then goes where it likes, `/auth/logged-out` by default | -| `/auth/check` **(planned)** | 204 or 401 (or 302 to login on request) for an ingress gating the application's pages; never a token or identity. See the [login gate](ingress.md#decision-2026-10-01-a-login-gate-for-the-applications-pages) | +| `/auth/check` | For an ingress's forward-auth: `204` when the browser is signed in and may make the request the ingress forwards (`X-Forwarded-Method`, with the CSRF rules of `/k8s` for a write), the `401` or `403` `/k8s` would give otherwise, or with `?redirect=true` a `302` to the login for a page load. With `?identity=true` the `204` carries `Krm-Foyer-Identity`, the API server's `userInfo` with the session's `displayName` and `connector`, for a domain backend. Never a token. See [the check](ingress.md#the-check) | | `/k8s/api/...` | Proxy core Kubernetes APIs after stripping `/k8s` | | `/k8s/apis/...` | Proxy grouped APIs, including CRDs and aggregated APIs | | `/k8s/api`, `/k8s/apis`, `/k8s/version`, `/k8s/openapi/...` | Proxy discovery and schema endpoints | diff --git a/docs/implementer-feedback.md b/docs/implementer-feedback.md index d76b147..9251fbf 100644 --- a/docs/implementer-feedback.md +++ b/docs/implementer-feedback.md @@ -41,6 +41,15 @@ as far as Voter is concerned. --- +## krm-foyer's answer (2026-10-05) + +| Entry | What changed | +| --- | --- | +| [1. Identity for a domain backend](#1-identity-for-a-domain-backend-on-the-same-origin) | `/auth/check` exists. It is a 204 or the refusal `/k8s` would give, judged on the request the ingress forwards, so a write to the backend needs the session's CSRF proof from krm-foyer's origin. With `?identity=true`, the 204 carries `Krm-Foyer-Identity`: the API server's `userInfo` with the session's `displayName` and `connector`, in one base64url JSON header, never the token. A SelfSubjectReview is reused per token for 30 seconds, within the session's request rate. The e2e suite forges the header through the real Traefik and checks the backend never sees it. See [the check](ingress.md#the-check) | +| [2. Room Pass's QR login](#2-prove-the-room-pass-qr-login-through-krm-foyer) | Run end to end (2026-10-06): Room Pass 2.0.0 and its own Dex with the `authproxy` connector in the e2e fixture, on a shared host with krm-foyer and a test application whose `/join-room` sets the join cookie, and a Chromium spec from the QR code to Room Pass's join page with the code already supplied, a display name, back to the application's page, `connector: room-pass` and the display metadata in `/auth/session`, a write with both extras in its audit event, a pod replacement and both logouts. It also checks the silent fallback, and that only Room Pass can reach Dex. Running it corrected [the recipe](room-pass.md): Room Pass's sign-out is a form on its `/join`, not a redirect to `/logout` | +| [3. Routing recipes](#3-routing-recipes-for-one-host-traefik-and-vite) | A Traefik `IngressRoute` with ForwardAuth middlewares, run by the e2e fixture beside its Gateway API routes; an nginx `auth_request` block, run by the fixture as the document has it since its login link carries the whole page, encoded (2026-10-06); a Vite dev-server proxy, in which Vite takes `task demo`'s public URL so cookies and CSRF work unchanged. The Vite recipe is not run by the suite. A redirect from the check is now absolute, because Traefik resolves a relative one against the check's address | +| [4. The operator as cluster-admin](#4-the-operator-signs-in-as-cluster-admin) | The browser identity recipe: one username expression gives krm-foyer's tokens (by `azp`, or the audience) a different name from the command line's, so `cluster-admin` stays on the command-line name. The fixture now does this, and a spec proves krm-foyer is refused what the command-line name was granted | + ## What 0.2.0 already covers | Voter today | krm-foyer 0.2.0 | Voter code that goes away | @@ -72,7 +81,7 @@ Room Pass, and none of the open entries below has been tried there. ## 1. Identity for a domain backend on the same origin -**open · ask · `/auth/check` with identity headers** +**done · `/auth/check?identity=true`, see [the check](ingress.md#the-check)** Some of Voter is domain logic, and it stays in a backend: @@ -112,7 +121,7 @@ this entry too. ## 2. Prove the Room Pass QR login through krm-foyer -**open · ask · release evidence, not new krm-foyer code** +**done · run end to end by the e2e suite, see [room-pass.md](room-pass.md#what-the-e2e-suite-runs)** The audience-release plan says: *"Before switching voter to foyer, prove one real QR-to-login journey there."* That has not happened. krm-foyer's fixture signs in with @@ -146,7 +155,7 @@ entry 3. ## 3. Routing recipes for one host: Traefik and Vite -**open · ask · roadmap: "Routing recipes for one shared domain"** +**done · Traefik run by the e2e fixture, Vite written: see [recipes](ingress.md#recipes) and [the dev-server proxy](ingress.md#during-development-a-dev-server-proxy)** Voter would serve four things from one host: @@ -168,7 +177,7 @@ expect to lose an afternoon. ## 4. The operator signs in as cluster-admin -**open question · [application scope](https://github.com/ConfigButler/krm-foyer/blob/main/docs/application-scope.md)** +**done · [a browser identity](application-scope.md#a-browser-identity-in-kubernetes), proved by one e2e spec** The operator logs in through GitHub as `github:simonkoudijs@gmail.com`, and that user is bound to `cluster-admin` (`auth/rbac/humans-rbac.yaml`). Today Voter's backend uses @@ -240,9 +249,13 @@ we can extract it as the example rather than have the guide written from scratch ## Suggested order -1. **Operator-only pilot, now.** It needs entry 3, and entry 4 decided. With the GitHub - login, Voter's admin and CoffeeConfig editor run through `/k8s` and `/stream/v1`. - This is the roadmap's own *"Voter's CoffeeConfig editor running on krm-foyer"*. +1. **Operator-only pilot, now.** It needs entry 3, entry 4 decided, and the + ValidatingAdmissionPolicy that keeps a CoffeeConfig save to `spec` (see + [What stays Voter's work](#what-stays-voters-work)): through `/k8s`, the editor's + `patch` grant also reaches labels and annotations, which gitops-reverser would + commit. With the GitHub login, Voter's admin and CoffeeConfig editor run through + `/k8s` and `/stream/v1`. This is the roadmap's own *"Voter's CoffeeConfig editor + running on krm-foyer"*. 2. **The audience login.** It needs entry 2. Participant login moves as one piece, since there can only be one session cookie on the origin. 3. **Votes and the storefront.** They need entry 1, or a decision in Voter to move diff --git a/docs/ingress.md b/docs/ingress.md index 1581b09..38e1849 100644 --- a/docs/ingress.md +++ b/docs/ingress.md @@ -105,9 +105,8 @@ spec: backendRefs: [{ name: workspaces-frontend, port: 8443 }] ``` -During development, the frontend's dev server plays the ingress's role: Vite's -`server.proxy`, for example, sends the four prefixes to a krm-foyer on a local cluster, -so the browser still sees one origin. The e2e fixture runs this recipe for real: +During development, the frontend's dev server plays the ingress's role (see the +[Vite recipe](#during-development-a-dev-server-proxy)). The e2e fixture runs this recipe for real: [gateway.yaml](../test/e2e/cluster/gateway.yaml) is a Gateway, an `HTTPRoute` per namespace and a `BackendTLSPolicy` per backend that makes the gateway verify its certificate, implemented by Traefik. @@ -123,6 +122,66 @@ Sending everything through krm-foyer would be wrong for several reasons: Paths outside its prefixes get a 404 from krm-foyer, which a unit test pins. +### During development: a dev-server proxy + +A frontend's development loop is its dev server, with hot reload. The dev server takes the +ingress's place: it serves the application's files, and proxies krm-foyer's four prefixes +to a krm-foyer on a local cluster, `task demo`'s for example. The browser still sees one +origin, so cookies and CSRF work as they do in production, with nothing mocked. + +The one rule: **the browser's origin is krm-foyer's public URL.** The session cookie is +`__Host-` and `Secure`, the login comes back to `/auth/callback`, and a write +needs an `Origin` equal to the public URL. A dev server on another origin breaks all +three, and rewriting `Origin` in the proxy to make writes pass would hide exactly the +bugs the loop should show. So the dev server takes the front door's address: +`https://foyer.localhost:8443`, `task demo`'s public URL, with the fixture's certificate +for that name (import `.e2e/ca.crt` into the browser, as for `task demo`). + +```ts +// vite.config.ts: the application's files from Vite, krm-foyer's prefixes from the +// task demo cluster, all on https://foyer.localhost:8443. +import { readFileSync } from 'node:fs' +import { Agent } from 'node:https' +import { defineConfig } from 'vite' + +const e2e = '../krm-foyer/.e2e' // a krm-foyer checkout after `task demo` +// FOYER_ADDR is krm-foyer's NodePort on the k3d node, reachable from where task demo +// ran; the front door's Traefik is not used. +const addr = /^FOYER_ADDR=(.*)$/m.exec(readFileSync(`${e2e}/foyer-env`, 'utf8'))![1] +const foyer = { + target: `https://${addr}`, + // Verify krm-foyer by the name its certificate holds, against the fixture CA. + agent: new Agent({ ca: readFileSync(`${e2e}/ca.crt`), servername: 'foyer.localhost' }), + secure: true, + // The browser's own Host, Origin and Cookie go through unchanged. + changeOrigin: false, +} + +export default defineConfig({ + server: { + host: 'foyer.localhost', + port: 8443, + strictPort: true, + https: { cert: readFileSync(`${e2e}/foyer/tls.crt`), key: readFileSync(`${e2e}/foyer/tls.key`) }, + proxy: { '/auth/': foyer, '/k8s/': foyer, '/stream/': foyer, '/_foyer/': foyer }, + }, +}) +``` + +Before starting Vite, stop the port-forward that puts Traefik on that port: +`pkill -f 'kubectl port-forward .*svc/traefik '` (`test/e2e/cluster/port-forward.sh` +brings it back). Dex stays on its own forward at `https://dex.localhost:5556`. + +What this leaves the application to decide is whether its own backend mock goes. Under +krm-foyer, `/k8s` and `/stream` are the real API server, so a mock of Kubernetes is no +longer needed for the loop. What remains to mock is the application's domain backend, if +it has one, and it sees the same `Krm-Foyer-Identity` only when something asks +`/auth/check` for it, which a dev server does not. Proxy it with a fixed identity +header in development, or run it behind the fixture's Traefik. + +This recipe is written from the fixture's facts (its public URL, certificate and +NodePort), but the e2e suite does not run Vite. + ### Why routing by path is safe when forward-auth is not Design A below is rejected because two programs parse the same path differently. Path @@ -201,48 +260,258 @@ Choose in this order: accepts the token once the backend's client ID is added to the issuer's audiences. Nobody shares or forwards credentials. The cost is two sessions, so logout has to end both. -3. **Later: krm-foyer answers an identity check** for the backend (option C below), - identity only. This avoids the second session, but the backend gets no token. +3. **krm-foyer answers an identity check** for the backend: [`/auth/check?identity=true`](#identity-for-a-domain-backend) + (option C below), identity only. This avoids the second session, but the backend gets + no token. krm-foyer never hands its tokens to another service. That would spread custody of the credential to code that krm-foyer does not test. ## Decision (2026-10-01): a login gate for the application's pages -**Planned, not implemented.** `/auth/check` currently returns 404. The behavior below -is the proposed contract; use the implemented `requireSession()` helper today. +**Built (2026-10-05), with identity for a domain backend** (option C below), because Voter +asked for both ([implementer feedback](implementer-feedback.md), entry 1). -The ingress can ask krm-foyer one question before serving the application's pages: *is -this browser signed in?* If it is not, the browser goes to the login, and comes back to -the page it asked for. Every frontend needs this, and it is wasteful to write it in each -one. It is a narrow use of forward-auth, and it is not the design rejected below. +The ingress can ask krm-foyer one question before serving a request: *is this browser +signed in, and may it make this request?* For a page, if it is not, the browser goes to +the login and comes back to the page it asked for. Every frontend needs this, and it is +wasteful to write it in each one. For a domain backend, the ingress can ask a second +question too: *who is it?* Both are narrow uses of forward-auth, and neither is the +design rejected below: the ingress never sends anything to the API server. + +### The check **`GET /auth/check`** answers: -- `204` when the session is valid; -- `401` when it is not; -- `302` to `/auth/login?return_to=` when it is not, if the request asks for it with - `?redirect=true`. +| Situation | Answer | +| --- | --- | +| Signed in, and the request may be made | `204`, with `Cache-Control: no-store` | +| With `?identity=true` | `204` and a `Krm-Foyer-Identity` header: see [identity](#identity-for-a-domain-backend) | +| No session | `401`, the interruption `/k8s` gives: a `Status` for code, a page with a **Sign in** link for a person | +| No session, `?redirect=true`, and a page load | `302` to `/auth/login?return_to=` | +| A write without the session's CSRF proof, or from another origin | `403`, `CSRFProofRequired` or `CrossOriginRequest`, as for `/k8s` | +| The API server cannot say who the user is (identity only) | Its own refusal, or `503`; never an identity guessed from the token | +| An option other than `redirect=true` and `identity=true`, or a bad `X-Forwarded-Method` | `400`, so that a typo such as `identity=1` fails loudly | + +The check is of **the request the ingress forwards**, not of the check's own request. +krm-foyer reads three things from it, all of which the browser could equally send to +krm-foyer itself, so none is trusted for more than that: + +- **The method,** from `X-Forwarded-Method`. A method other than `GET` or `HEAD` is a + write, and needs what a write to `/k8s` needs: the session's CSRF token in + `X-CSRF-Token`, and `Origin` (or `Sec-Fetch-Site: same-origin`) naming krm-foyer's + origin. Without the header the check is of a `GET`. Methods are case-sensitive, so + `get` is a write. **The ingress must set this header itself** (Traefik always does; + the nginx recipe below sets it), or a browser's own copy decides how a write is checked. +- **The page,** from `X-Forwarded-Uri` (Traefik) or `X-Original-URI` (nginx). It is + validated as a local path exactly like `return_to`, and anything else becomes `/`. It + only decides where the login returns to. +- **The browser's own headers:** `Cookie`, `Origin`, `X-CSRF-Token`, and the + `Sec-Fetch-*` headers that decide between a page and a `Status`, and between a + redirect and a `401`. Only a page load (no `Sec-Fetch-Mode`, or `navigate`, on a `GET`) + is redirected; a script gets the `401` it can act on. + +### Identity for a domain backend + +With `?identity=true`, a `204` carries one header, `Krm-Foyer-Identity`: the unpadded +base64url encoding of this JSON. + +```json +{ + "userInfo": { + "username": "oidc:alice@example.com", + "uid": "...", + "groups": ["oidc:voters", "system:authenticated"], + "extra": { "configbutler.ai/claims/display-name": ["Alice"] } + }, + "displayName": "Alice", + "connector": "room-pass", + "issuer": "https://dex.example.com", + "expiresAt": "2026-10-05T20:00:00Z" +} +``` + +- `userInfo` is **the API server's answer** to a SelfSubjectReview sent with the user's own + token, exactly as [`/auth/whoami`](design.md#whoami) shows it: the name RBAC and the audit + log use, with the groups and extras the cluster's authentication configuration + maps. A backend that records who did something records the same name Kubernetes does. +- `displayName` and `connector` are the session's, as `/auth/session` shows them + ([session claims](design.md#session-claims)). `connector` is present only when one is + configured. A backend can, for example, accept votes only from `room-pass` logins. +- **Never a token.** The backend learns who the user is; it cannot act as them in + Kubernetes. A backend that needs that is a separate design question. + +One header, not one per field. An ingress copies it whole, a backend parses one value, +and group names may hold commas, which a list in a header cannot carry safely. + +A check with identity goes through the same [bounds](bounds.md) as a request to `/k8s`: +it counts against the session's request rate. The API server's answer is reused for 30 +seconds per token (never past the session's end), so a backend checked on every request +costs one SelfSubjectReview per user per 30 seconds. The answer depends only on the +token and the cluster's authentication configuration, so 30 seconds is how long a change +to that configuration can take to reach a backend. **Access decisions are not reused or +made here:** what the user may do in Kubernetes is still the API server's decision on +every `/k8s` request, and what the user may do in the backend is the backend's. + +What the deployment must get right, because krm-foyer cannot check it: + +- **The ingress removes the browser's copy of `Krm-Foyer-Identity`** before it adds + krm-foyer's. Traefik's ForwardAuth does this for every header in + `authResponseHeaders`; the nginx recipe below overwrites it. The e2e suite sends a + forged one through Traefik and checks the backend never sees it. +- **Only the ingress reaches the backend,** on a route that always asks the check, so + that nothing can send it a header the check did not write. A NetworkPolicy admitting only + the ingress to the backend's port is the way to enforce that. Room Pass contained its + own header trust the same way. +- **The backend's route asks with `identity=true`.** A page-gate route (no identity) + does not remove the header, so a backend behind it would believe whatever the browser + sent. + +### Recipes + +The e2e fixture runs the Traefik recipe for real +([traefik-routes.yaml](../test/e2e/cluster/traefik-routes.yaml)), beside its Gateway API +routes on the same origin: `/public/` is a domain backend behind the identity check, and +`/members/` pages behind the login gate. Its specs try a forged identity header, writes +without CSRF proof and from another origin, a signed-out page load and a signed-out script, +all through the real Traefik. + +**Traefik, `IngressRoute` and ForwardAuth.** For a cluster that routes with Traefik's own +resources, as Voter's does. krm-foyer's own prefixes stay on a plain route (or the +`HTTPRoute` above); only the application's routes get a middleware. -It never returns a token or identity headers. The page the user wanted arrives in the -ingress's original-URI header (`X-Original-URI` in nginx, `X-Forwarded-Uri` in Traefik). -It is validated as a local path exactly like `return_to`, and anything else becomes `/`. +```yaml +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: { name: foyer-identity } +spec: + forwardAuth: + # krm-foyer's Service, by a name its certificate holds, verified against its CA. + address: https://krm-foyer.krm-foyer.svc/auth/check?identity=true + tls: { caSecret: krm-foyer-ca } + # Removes the browser's copy, then copies krm-foyer's. + authResponseHeaders: [Krm-Foyer-Identity] + trustForwardHeader: false +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: { name: foyer-gate } +spec: + forwardAuth: + address: https://krm-foyer.krm-foyer.svc/auth/check?redirect=true + tls: { caSecret: krm-foyer-ca } + trustForwardHeader: false +--- +# The backend and the file server have no use for the session cookie. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: { name: no-cookie } +spec: + headers: + customRequestHeaders: { Cookie: "" } +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: { name: voter } +spec: + entryPoints: [websecure] + routes: + # krm-foyer's prefixes, whole segments: Traefik's PathPrefix is a plain string + # prefix, so /stream would also catch /streams. A Service in another namespace + # needs the CRD provider's allowCrossNamespace; or keep this route beside krm-foyer. + - match: Host(`voter.example.com`) && (PathPrefix(`/auth/`) || PathPrefix(`/k8s/`) || PathPrefix(`/stream/`) || PathPrefix(`/_foyer/`)) + services: [{ name: krm-foyer, port: 443, scheme: https, serversTransport: krm-foyer }] + - match: Host(`voter.example.com`) && PathPrefix(`/public/`) + middlewares: [{ name: foyer-identity }, { name: no-cookie }] + services: [{ name: voter-backend, port: 8080 }] + # Pages behind the login gate. Files the pages load (scripts, images) stay public. + - match: Host(`voter.example.com`) && PathPrefix(`/admin/`) + middlewares: [{ name: foyer-gate }, { name: no-cookie }] + services: [{ name: voter-web, port: 8080 }] + - match: Host(`voter.example.com`) + middlewares: [{ name: no-cookie }] + services: [{ name: voter-web, port: 8080 }] + tls: { secretName: voter-tls } +``` + +Traefik sends the check a `GET` with the browser's headers and its own `X-Forwarded-Method` +and `X-Forwarded-Uri`, and passes a non-2xx answer to the browser as it is: the `401` +`Status` to a script, the sign-in page or the `302` to a page load. Its `rule`s are +matched by length unless `priority` says otherwise; give the gated routes a priority when +they share an entry point with Gateway API routes, as the fixture does. + +**nginx, `auth_request`.** nginx accepts only 2xx, 401 and 403 from the check, and turns +the 401 into the login itself, so it does not use `redirect=true`. The page goes into +the login link as one `return_to` value, so it has to be encoded, and stock nginx has +no directive that encodes a variable for a query. The recipe uses +[njs](https://nginx.org/en/docs/njs/), which the official nginx images include: load it +at the top of `nginx.conf` with `load_module modules/ngx_http_js_module.so;`. + +```nginx +# The page the browser asked for, encoded as one query value (foyer.js, below). +js_import foyer from /etc/nginx/foyer.js; +js_set $foyer_return_to foyer.returnTo; + +location = /_check { + internal; + proxy_pass https://krm-foyer.krm-foyer.svc/auth/check?identity=true; + proxy_ssl_verify on; + proxy_ssl_trusted_certificate /etc/nginx/krm-foyer-ca.crt; + proxy_ssl_name krm-foyer.krm-foyer.svc; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + # Set here, so the browser's own copies count for nothing. + proxy_set_header X-Forwarded-Method $request_method; + proxy_set_header X-Original-URI $request_uri; +} + +location /public/ { + auth_request /_check; + auth_request_set $foyer_identity $upstream_http_krm_foyer_identity; + # Overwrites the browser's copy, with krm-foyer's or with nothing. + proxy_set_header Krm-Foyer-Identity $foyer_identity; + proxy_set_header Cookie ""; + proxy_pass http://voter-backend:8080; +} + +location /admin/ { + auth_request /_check; + error_page 401 = @login; + proxy_set_header Cookie ""; + proxy_pass http://voter-web:8080; +} + +location @login { + # Relative, so the browser stays on the origin it used, whatever port nginx has. + absolute_redirect off; + return 302 /auth/login?return_to=$foyer_return_to; +} +``` -Ingresses differ in what they do with a refusal, which is why there are two refusals: +```js +// foyer.js: the raw request target, path and query, as one query value. Every byte +// that would end the value or change its meaning (& = + # % and the rest) is +// percent-encoded, so krm-foyer decodes exactly the target the browser sent. +function returnTo(r) { + return encodeURIComponent(r.variables.request_uri); +} -- **nginx `auth_request`** accepts only 2xx, 401 and 403 from the check. It turns the 401 - into the redirect itself, with `error_page 401 = @login`. -- **Traefik ForwardAuth and Envoy's external authorization** pass the check's response to - the browser, so they use `?redirect=true`. +export default { returnTo }; +``` -Gateway API introduced an experimental `externalAuth` filter in -[v1.4](https://kubernetes.io/blog/2025/11/06/gateway-api-v1-4/); support depends on the -installed CRDs and gateway implementation. This fixture does not configure it. -Without a usable page gate, the browser helper's -`requireSession()` does the same at page load, in a few lines of JavaScript. The gate is -an improvement for whoever has it, not a requirement. +`$request_uri` is the raw request target, so the page comes back exactly as the +browser asked for it, its own encoding included. Put in the query as it is, as an +earlier version of this recipe did, it would not: the page's `&` starts a parameter of +the login's (`/admin/edit?name=x&tab=history` came back as `/admin/edit?name=x`), its +`%26` and `+` are decoded once too often, and a parameter of the page's named +`return_to` or `oidc.login_hint` becomes the login's own. krm-foyer's login checks the +decoded value as a local path and refuses anything else, so it cannot send the browser +off-site. The e2e fixture runs this recipe as written here +([nginx-door.conf](../test/e2e/cluster/nginx-door.conf) takes it from this page), in +front of krm-foyer, and walks a login from a gated page with several parameters, with +percent-encoding and a literal `+`, and with parameters named like the login's own. -How to deploy it: +### How to deploy it - **Gate page routes only.** JavaScript bundles and images are public by nature; gating them only turns an expired session into broken asset loads. @@ -359,7 +628,8 @@ handler, with no change to `/k8s`. It still creates a contract of its own: - If the token were included, every such backend would hold Kubernetes credentials. krm-foyer would then share custody of the credential with services it does not test. -**Later, and identity only.** Build it when a hybrid application asks, on top of +**Built, identity only (2026-10-05),** when Voter, a hybrid application, asked: +[identity for a domain backend](#identity-for-a-domain-backend), on top of `/auth/check`, and without the token. A domain backend that needs to act on Kubernetes as the user is a separate design question. @@ -374,5 +644,5 @@ auth proxy would make krm-foyer less opinionated, much larger, and impossible to across all of those combinations. The flexibility people need is in the network, not in who decides. Both TLS models are -supported, a transparent ingress is fine, the planned login gate can cover page loads, and +supported, a transparent ingress is fine, the login gate can cover page loads, and the seam for external login exists if it is ever needed. diff --git a/docs/investigations/rollout-connections.md b/docs/investigations/rollout-connections.md new file mode 100644 index 0000000..920b25a --- /dev/null +++ b/docs/investigations/rollout-connections.md @@ -0,0 +1,92 @@ +# Connections across a pod replacement + +**Status (2026-10-06): the rehearsal's dial timeouts are diagnosed and fixed in the e2e +harness. A rollout that swallows no connection is parked:** live rollouts are not part of +the demo, and it is operator work the project does not need yet. This page records what +was measured, so the work can start from here rather than from scratch. +The roadmap keeps the item ("Rolling updates that refuse no connection"). + +## What failed + +A fresh `task verify` run failed in the rehearsal's spec "with a watch of each user's own", +at `rehearsal_test.go:144`, while it opened 1,800 streams at once (200 identities, 9 +each): `dial tcp 172.29.250.2:30443: i/o timeout`. The same spec passed when run alone. + +## What it was + +It was not krm-foyer refusing anything, and not a shortage of capacity. The fixture's +NodePort path takes a burst of 8,000 simultaneous connections with no loss (p99 162 ms). +The timeouts appear only after a krm-foyer pod has been replaced: + +1. A connection is tracked on the node (conntrack) together with the pod kube-proxy sent + it to. When that pod goes away, kube-proxy changes its rules for new connections, but + it does not remove TCP entries already tracked. It does remove UDP ones. The entries + live on until they time out, up to two minutes for `SYN_SENT`. +2. SYNs sent in the moment between the old pod's address going away and kube-proxy's + rules following it are never answered. Their entries still point at the gone address. +3. A later connection from the same client address and port follows such an entry to + the gone address. Nobody answers it or its retransmissions, so the dial times out + after five seconds. + +The suite is one client making thousands of connections a minute to one NodePort, so it +reuses source ports quickly. The rehearsal replaces krm-foyer twice per run, and other +specs replace it too, once with a forced delete. Whether a burst met stale entries +depended on what ran in the two minutes before it, which explains why the failure came +and went with spec order. + +Evidence, from the fixture: + +| Check | Result | +| --- | --- | +| 1,800 dials at once, steady state | 0 failures, repeatedly; 8,000 at once also 0 | +| After a `rollout restart` with traffic during it, 3 bursts of 1,800 | 16 to 28 `i/o timeout` per burst; 44 and 239 per three bursts in two later rounds | +| After a forced pod delete, the same | 75 to 97 per burst; 229 and 172 per three bursts | +| Stuck client sockets compared with the node's `conntrack -L` | Their source ports matched `SYN_SENT` entries DNATed to deleted pods, one of them deleted two minutes earlier | +| The same replacements, then the cleanup below, then the bursts | 0 timeouts in every round | + +## The fix in the harness + +After the suite replaces krm-foyer's pod, `fixture.forgetGonePods` deletes the node's +conntrack entries for krm-foyer's NodePort whose reply comes from a pod address that no +longer exists ([fixture_test.go](../../test/e2e/fixture_test.go), called from +`replaceFoyer`). This is the cleanup kube-proxy already does for UDP. It touches only +this fixture's node and only entries that can never be answered. The rehearsal's +workload, 200 identities × 9 streams opened at once, and its 5-second dial timeout are +unchanged. + +## What is parked: a rollout that swallows no connection + +The same mechanism affects a client connecting *during* a rollout, and a real client +behind one NAT address could meet it too. The pending spec +[foyer_rollout_test.go](../../test/e2e/foyer_rollout_test.go) dials continuously across a +`rollout restart` and accepts refusals, but no timeouts. What was tried, uncommitted: + +- **A shutdown delay in krm-foyer.** On SIGTERM it would fail `/readyz` at once and keep + serving for `-shutdown-delay`, with a 1-second readiness probe, so the endpoint stops + serving before the address goes away. Measured after a `rollout restart`: the old + endpoint stopped serving at 2.7 to 5.1 s, its NAT rule was gone at 4.0 to 5.8 s, and the + pod itself was gone at 5.8 to 7.8 s without a delay, or 12 to 13 s with a 10-second + delay. With the delay, a rollout still swallowed 0 to 2 connections in about one run + in two. The cause of those was not found. +- **Probe design matters.** A probe that sends `Connection: close` makes krm-foyer close + first, often with a reset. The client then reuses those ports at once, against the + node's fresh `CLOSE` entries: every SYN of a burst right after that was delayed by one + retransmission (1.09 s). The parked spec therefore only opens and closes TCP + connections. +- **Two explanations were ruled out.** Entries in `LAST_ACK` expire before the client + frees their ports. `TIME_WAIT` entries from connections the client closed are reopened + correctly by conntrack. + +The chart's `Recreate` strategy keeps one pod at a time, so a rollout always has a gap +in which new connections are refused. "Refuse no connection" needs two pods during an +update, which in turn needs the multi-replica design the roadmap lists first. + +## When this comes back + +- Start from the pending spec and the measurements above, on the fixture as it is. +- Decide first whether the goal is "refuse, never swallow" (one pod) or "refuse nothing" + (two pods, after the multi-replica design). +- gitops-reverser's move to Kubernetes 1.37 ran its e2e on one server and one agent + after a server restart left cluster DNS dead. That is a different failure, but the + fixture's own server restart (when the authentication configuration changes) would + benefit from the same layout. diff --git a/docs/roadmap.md b/docs/roadmap.md index 6001b01..cd30411 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -75,8 +75,10 @@ new design only when multiple replicas or stronger revocation guarantees are req Add docs/link checks early enough to prevent status drift; keep broader coverage targets secondary to tests of specific failure modes. -The page login gate (`/auth/check`) is a convenience after the deployment and integration -work: `requireSession()` already handles arrival while signed out. Application scopes, +The login gate and identity for a domain backend (`/auth/check`) exist since Voter +asked for them, with a browser identity proved in the cluster's authentication +configuration as the first step of application scope; see +[implementer feedback](implementer-feedback.md). A scope list in krm-foyer, external-login modes and upgrade protocols stay driven by actual adopter requirements. ## Completed milestones @@ -125,7 +127,10 @@ Security items need tests that try to get past the boundary. - [x] The implemented `foyer` authentication/access specs: login, identity, differential answers, RBAC changes, watches, path and subresource refusals, CSRF, logout and the token scan, each checked by deploying a krm-foyer broken on purpose -- [ ] The remaining `foyer` specs: refusal of a refresh, and dedicated ingress/login-gate behavior +- [x] Login gate and identity check through the real Traefik: a forged identity header, + writes without CSRF proof and from another origin, signed-out page loads and scripts +- [ ] The remaining `foyer` specs: refusal of a refresh, and spoofed `Host` and + `X-Forwarded-*` through the ingress - [x] Browser e2e: log in, read, create, edit, get refused with 403, hit a 409, log out, and no credential within the page's reach. Chromium driven from the Go suite with chromedp, keeping browser e2e in Go. Node separately runs the helper unit @@ -154,9 +159,10 @@ Security items need tests that try to get past the boundary. whatever the method; a test that a cross-site `GET` navigation is refused before it reaches the backend -- [ ] [Application scope](application-scope.md): first document a browser identity in the - cluster's authentication config, proved by one e2e spec; then a scope list in - krm-foyer for clusters where that is not possible +- [x] [Application scope](application-scope.md), first step: a browser identity in the + cluster's authentication config, documented and proved by one e2e spec +- [ ] Application scope, then: a scope list in krm-foyer for clusters where the + authentication config cannot change ### Deployment @@ -171,15 +177,22 @@ Security items need tests that try to get past the boundary. - [ ] Helm chart values for both models - [ ] Rolling updates that refuse no connection: krm-foyer stops listening as soon as it is told to stop, while its Service may still route to it for a moment, so a - rollout refuses connections briefly (seen by the rehearsal, which restarts it). A - wait before shutdown, or readiness turned off first, with a test -- [ ] Routing recipes for one shared domain: a Gateway API `HTTPRoute`, an nginx server - block and a Vite dev-server proxy + rollout refuses connections briefly, and now and then leaves one unanswered. + Parked (2026-10-06), not needed for the demo: the measurements, what was tried and + a pending spec are in + [rollout-connections.md](investigations/rollout-connections.md) +- [x] Routing recipes for one shared domain: a Gateway API `HTTPRoute`, a Traefik + `IngressRoute` and an nginx server block (all run by the e2e fixture, nginx's + copied from the document), and a Vite dev-server proxy (written, not run). See + [ingress](ingress.md) - [x] `requireSession()` in the browser helper for navigation to login -- [ ] Login gate: `GET /auth/check` for an ingress gating the application's pages, with +- [x] Login gate: `GET /auth/check` for an ingress gating the application's pages, with nginx and Traefik recipes -- [ ] Later, when a hybrid application asks: identity headers from the check for a domain - backend, never the token +- [x] Identity from the check for a domain backend, never the token + (`?identity=true`, `Krm-Foyer-Identity`), asked for by Voter +- [x] Room Pass's QR login run end to end through krm-foyer: Room Pass 2.0.0 and its + Dex in the fixture, and a Chromium spec from the QR code to logout, with the + release checks Voter's plan asks for. See [room-pass.md](room-pass.md) ### Access diff --git a/docs/room-pass.md b/docs/room-pass.md new file mode 100644 index 0000000..ffec32b --- /dev/null +++ b/docs/room-pass.md @@ -0,0 +1,190 @@ +# Recipe: Room Pass's QR login through krm-foyer + +How an application on krm-foyer signs in an audience with +[Room Pass](https://github.com/sunib/room-pass)'s QR join: scan a code, type a display +name, and arrive signed in. Voter asked for this in the +[implementer feedback](implementer-feedback.md) (entry 2), as the release evidence its +audience-release plan requires before switching to krm-foyer. + +**Status: run end to end (2026-10-06)** against Room Pass 2.0.0's released image +(`ghcr.io/sunib/room-pass:2.0.0`, revision `2185782`, pinned by digest) and its CRDs, +behind its own Dex, in the e2e fixture, by a real Chromium +([room_pass_test.go](../test/e2e/room_pass_test.go), fixture in +[room-pass.sh](../test/e2e/cluster/room-pass.sh)). [What runs](#what-the-e2e-suite-runs) +lists each check. Running it corrected three details of the first version of this page: +how Room Pass signs out, which the recipe got wrong; the code's `SameSite=Lax`, which a +typed URL does not test; and the display name, which arrives folded. + +## The journey + +```text +QR code ─► /join-room?code=K7Q2 the application's endpoint, on krm-foyer's host + sets __Host-room-pass-joincode=K7Q2, then 302 ─► + /auth/login?return_to=/room&oidc.connector_id=room-pass krm-foyer + ─► Dex (Room Pass's issuer host) /auth?connector_id=room-pass + ─► Room Pass /join on krm-foyer's host: the code is filled in from the + cookie, the participant types a display name + ─► Dex ─► /auth/callback (krm-foyer) ─► 303 /room + /auth/session: {"displayName": "Ada-Lovelace", "groups": ["demo:my-talk"], "connector": "room-pass"} +``` + +The display name is the one Room Pass stores: what the participant typed, folded to +letters, digits and dashes (`Ada Lovelace` becomes `Ada-Lovelace`), as its join page +shows before they continue. The email is made from it, `ada-lovelace@koudijs.dev.test`. + +Three programs share one host: the application, krm-foyer, and Room Pass's `/join`, +`/bind` and `/logout`. That is the point most likely to break. Room Pass's join cookie +reaches `/join` only when the application set it on the same host +(`__Host-` cookies have no `Domain`), and with `SameSite=Lax`. If either is wrong, the +flow does not fail. It quietly falls back to asking the participant to type the code. +Only a browser test catches that, and only one that opens the QR URL the way a scanner +hands it on: from a link, another site's navigation. A URL typed into the address bar, +or opened by a test driver's own navigation, carries even a `Strict` cookie through the +issuer's redirects, so it passes with the mistake Room Pass warns about. The e2e spec +follows a link from another site, and fails with `Strict` (checked by changing it). + +## The join endpoint stays the application's + +The join code does not go through krm-foyer. Room Pass builds the join URL after Dex, so +no `oidc.*` login parameter could carry it, and the code is not krm-foyer's business. +The application keeps a small endpoint that follows Room Pass's +[cookie contract](https://github.com/sunib/room-pass/blob/main/docs/qr-join.md): + +- the value is the room code, bounded before it is set: upper-case `A-Z0-9`, at most 12 + characters (Room Pass ignores anything else); +- `Secure`, `HttpOnly`, `Path=/`, **`SameSite=Lax`**: `Strict` breaks the flow, because + the cookie must survive the redirects through the issuer host; +- a lifetime of a few minutes; `/join` expires it when it uses it; +- never logged, and never in the authorization request. The code is in the QR URL's + query, so that includes the edge's access log for that route: in Traefik, + `observability: {accessLogs: false}` on its own route; in nginx, `access_log off` in + its `location`. + +```go +// GET /join-room?code=K7Q2: the QR code's target. Voter's is voter/oidc.go:401-484. +func joinRoom(w http.ResponseWriter, r *http.Request) { + code := strings.ToUpper(r.URL.Query().Get("code")) + if len(code) == 0 || len(code) > 12 || strings.Trim(code, "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") != "" { + http.Error(w, "not a room code", http.StatusBadRequest) + return + } + http.SetCookie(w, &http.Cookie{ + Name: "__Host-room-pass-joincode", Value: code, Path: "/", + Secure: true, HttpOnly: true, SameSite: http.SameSiteLaxMode, MaxAge: 300, + }) + // The destination is the application's own, fixed here, never from the request. + http.Redirect(w, r, "/auth/login?return_to=%2Froom&oidc.connector_id=room-pass", http.StatusFound) +} +``` + +Room Pass's `cmd/room-qr` points QR codes at `LOGIN_PATH`, `/auth/login` by default. Set +it to this endpoint: krm-foyer's `/auth/login` takes `return_to`, not `code` or `return`. + +## krm-foyer's configuration + +```yaml +# Helm values +oidc: + issuer: https://login.example.com # Room Pass's issuer host: Dex behind Room Pass + clientID: voter + # federated:id makes Dex add federated_claims.connector_id; groups carries the + # Room's audience group. Add audience:server:client_id:kubernetes if the API server + # accepts that audience rather than this client's. + scopes: [openid, email, profile, groups, "federated:id"] +login: + authorizationParameters: + connector_id: + allowFromRequest: true + allowedValues: [room-pass, github] # the audience, and the operator + sessionClaims: + connector: /federated_claims/connector_id # name and groups are the defaults +``` + +In Room Pass's Dex, krm-foyer is a static client with the redirect URI +`https:///auth/callback`, and Room Pass's `ALLOWED_RETURN_URLS` (and the Room's +`allowedReturnURLs`) name `https:///`; `/join` refuses with 400 otherwise. +Room Pass's `JOIN_ORIGIN` is the shared host, port included, since Room Pass routes by +`Host`. + +The ID token from a Room Pass login carries `name` (the display name), `email` +(`@`, verified), `groups` (the Room's `audienceGroup`, such as +`demo:my-talk`) and `federated_claims.connector_id: room-pass`. Room Pass's reference +API-server configuration (`deploy/apiserver/authentication-config.yaml`) maps the +username to `demo:`, requires `connector_id in ['room-pass']`, and requires every +group to start with `demo:`. The e2e fixture uses those rules, and also requires the +token's client to be krm-foyer, by the rules of the +[browser identity](application-scope.md#a-browser-identity-in-kubernetes). Whatever the +mapping, `/auth/whoami` and `Krm-Foyer-Identity` show the name it produced. + +## The edge: one host, four owners + +| Path on the shared host | Goes to | +| --- | --- | +| `/auth/`, `/k8s/`, `/stream/`, `/_foyer/` | krm-foyer | +| `/join`, `/bind`, `/logout` (exact) | Room Pass | +| `/join-room` | The application's join endpoint | +| `/public/` | The application's domain backend, behind [`/auth/check?identity=true`](ingress.md#identity-for-a-domain-backend) | +| everything else | The application's files | + +Room Pass's issuer host (`login.example.com`) goes to Room Pass entirely; it forwards +only Dex's protocol paths to Dex, and nothing may expose Dex's Service directly. The +edge strips `X-Remote-*` from every request it sends Room Pass, as Room Pass's own +`deploy/edge/traefik/middlewares.yaml` does: Dex's `authproxy` connector believes those +headers. A NetworkPolicy lets only Room Pass's pods reach Dex. In Traefik that is the +[IngressRoute recipe](ingress.md#recipes) plus Room Pass's routes; the fixture's are in +[routes.yaml](../test/e2e/cluster/room-pass/routes.yaml). The API server, and krm-foyer, +fetch the issuer's discovery and keys at the issuer's host too, so it has to resolve +and be reachable from the control plane and from pods, not only from browsers. + +**Logout is two programs, each with its own contract.** + +- krm-foyer's `POST /auth/logout`, with the session's CSRF proof, clears krm-foyer's + session cookie and ends the session's open streams. +- Room Pass's `/logout` accepts only a `POST` with Room Pass's own CSRF token, which only + its join page holds: a redirect or a link to it is refused with 403. So after + krm-foyer's logout, send the browser to `/join`. An enrolled participant sees + "You're already enrolled as …" and a **Sign out of this browser** button, which posts + to `/logout`, clears Room Pass's `__Host-rp-session` and returns to `/join`, now asking + for a code. + +Neither revokes anything. A copy of krm-foyer's cookie taken before its logout is still +a session until it expires ([Sessions](design.md#sessions)), and the e2e spec checks +exactly that. Room Pass's session cookie is sealed the same way, and the participant +stays enrolled, so the name they chose stays taken: signing in again means joining +under another name. An ID token already issued stays valid at the API server until it +expires. To shut a participant out, revoke the Participant in Room Pass and let the +token's lifetime pass, or end the Room. + +## What the e2e suite runs + +[room_pass_test.go](../test/e2e/room_pass_test.go) drives Chromium through the +fixture's two hosts, `room.localhost` (the application, krm-foyer-room and Room Pass's +`/join`) and `room-pass.localhost` (the issuer), both through Traefik. The fixture +([room-pass.sh](../test/e2e/cluster/room-pass.sh), [room-pass/](../test/e2e/cluster/room-pass/)) +has Room Pass and its Dex with the `authproxy` connector, the CRDs and a Room, cookie +keys and RBAC, certificates from the fixture CA, the API server trusting the issuer as a +third JWT issuer under Room Pass's reference rules, the routes of the table above, and +a test application whose `/join-room` implements the cookie contract in nginx. Nothing +Room Pass's is in krm-foyer: krm-foyer-room is the chart with the generic settings +above ([foyer-room-values.yaml](../test/e2e/cluster/foyer-room-values.yaml)). + +| Release check | What the spec does | +| --- | --- | +| The QR journey | Takes the Room's current code as `cmd/room-qr` does, from its status, and follows `https://room.localhost:8443/join-room?code=…` from a link on another site. Expects Room Pass's join page on the shared host with the code shown as scanned and no code field, and the code in no URL but the QR code's own: not in the authorization request, not in Room Pass's handoff URLs. Types a display name and expects to arrive at `/room/` | +| The silent fallback | The same login without the join cookie shows the code field, so the check above can fail. Changing the cookie to `SameSite=Strict` makes it fail with "Room Pass asked for the room code" | +| Display metadata | `/auth/session` has `connector: room-pass`, the folded display name, the made-up email and the Room's audience group | +| A write attributed through both extras | `/auth/whoami` shows `demo:`, never the typed name, the audience group and both extras. A ConfigMap created from the page with the session's CSRF proof, under a grant to the audience group, has both extras in its audit event, and no impersonation. A Secret, not granted, is the API server's 403 naming that user | +| A pod replacement keeps the session | krm-foyer-room is restarted; the browser's session, its claims and a write still work, with no new login | +| Logout | krm-foyer's `POST /auth/logout`, then Room Pass's sign-out form on `/join`; each cookie is gone, and a new scan asks for a display name again. A copy of krm-foyer's cookie taken before logout still answers `/auth/session`: logout revokes nothing | +| No credential leaks | No page script sees a cookie or storage; no response header or URL the browser saw, nor `/auth/session` or `/auth/whoami`, holds a JWT; krm-foyer-room's, Room Pass's and the application's logs hold no JWT, no cookie value and no join code | + +A second spec checks the `authproxy` boundary. From a pod labelled as Room Pass in its +namespace, Dex answers; from a pod with another label there, or with Room Pass's label in +another namespace, the connection is refused: k3s enforces the NetworkPolicy, and +deleting it makes the spec fail. Through the edge, a login followed with forged +`X-Remote-*` headers on every request ends at Room Pass's join form and never at +krm-foyer's callback, and Room Pass's callback and completion paths refuse such a request +outright. + +Not run: a phone. The browser is Chromium, which is what Room Pass's own browser test +uses too; Safari's handling of `SameSite` and of a camera-opened URL is not covered. diff --git a/docs/testing.md b/docs/testing.md index 970b57a..3c32b73 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -360,9 +360,12 @@ single-node k3d cluster on it, then deploys the two issuers into the cluster ([issuers.yaml](../test/e2e/cluster/issuers.yaml)). The API server trusts both through an [AuthenticationConfiguration](../test/e2e/cluster/authentication-config.yaml), under the same rules, and records requests with an [audit policy](../test/e2e/cluster/audit-policy.yaml). -The API server is not a pod and cannot use cluster DNS: k3d's `--host-alias` puts each -issuer's name in the node's `/etc/hosts`, pointing at its Service's fixed ClusterIP, and -in CoreDNS for pods, so every caller uses the same issuer URL. Dex keeps its state in +The API server is not a pod and cannot use cluster DNS: the script puts each issuer's +name in the node's `/etc/hosts`, pointing at its Service's fixed ClusterIP, and in a +CoreDNS server block for pods (the `coredns-custom` ConfigMap, which k3s's Corefile +imports), so every caller uses the same issuer URL. It writes `/etc/hosts` on every run, +because Docker rewrites it when the node restarts, which a changed authentication +configuration causes. Dex keeps its state in custom resources, its signing keys included: with memory storage, a restarted Dex signs with new keys, and the API server refused every token for 221 seconds before it fetched them. The devcontainer joins the network, and a CI runner is the Docker host, so both reach @@ -401,7 +404,18 @@ an `HTTPRoute` sending `/` to the file server without the `Cookie` header, one s `/auth`, `/k8s`, `/stream` and `/_foyer` to krm-foyer, and a `BackendTLSPolicy` for each backend under which Traefik verifies its certificate (with a wrong hostname in it, every request fails). The file server answers 400 to any request that still carries a cookie, -so every signed-in browser spec fails if the route stops removing it. Then +so every signed-in browser spec fails if the route stops removing it. It also applies +[traefik-routes.yaml](../test/e2e/cluster/traefik-routes.yaml), the Traefik recipe of +[the check](ingress.md#the-check): an `IngressRoute` on the same host with ForwardAuth +middlewares to `/auth/check`, which Traefik reaches at krm-foyer's Service name (its +certificate holds both names). `/public/whoami` stands in for a domain backend and +echoes the `Krm-Foyer-Identity` it received, and `/members/` is a page behind the login +gate. Specs reach them with `fx.frontDoorBrowser()`, through Traefik. The nginx +recipe of the check runs too, as the document has it: front-door.sh copies its blocks +out of [ingress.md](ingress.md#recipes) into +[nginx-door.conf](../test/e2e/cluster/nginx-door.conf), an nginx in front of krm-foyer +on a NodePort of its own, under the same public name, whose two backends echo what they +receive. Specs reach it with `fx.nginxBrowser()`. Then [port-forward.sh](../test/e2e/cluster/port-forward.sh) forwards Traefik to `127.0.0.1:8443` and Dex to `127.0.0.1:5556` in this container, detached, and checks both by their public names. Browsers resolve `foyer.localhost` and `dex.localhost` to loopback, @@ -410,14 +424,32 @@ and VS Code forwards both ports to the machine the browser runs on, keeping thei wherever Docker runs. A port-forward follows one pod: when Dex or Traefik rolls, run `test/e2e/cluster/port-forward.sh` (or `task e2e-deploy`) again. +[room-pass.sh](../test/e2e/cluster/room-pass.sh) adds Room Pass's QR login +([room-pass.md](room-pass.md)) on two more hosts through the same Traefik: +`room.localhost`, which krm-foyer-room (a third release of the chart, signing in at Room +Pass's Dex), Room Pass's `/join`, `/bind` and `/logout`, and a test application with the +QR entry point `/join-room` share; and `room-pass.localhost`, Room Pass's issuer. Room +Pass 2.0.0 and its own Dex, with the `authproxy` connector, a NetworkPolicy that only +Room Pass passes, SQLite on a volume, Room Pass's CRDs and a Room, all in +[room-pass/](../test/e2e/cluster/room-pass/). The API server trusts that Dex as a third +JWT issuer, which it and krm-foyer-room reach at `room-pass.localhost:8443`: a Service +with a fixed address in front of Traefik. Everything Room Pass's lives in the fixture; +krm-foyer-room has only the chart's generic login settings. A browser reaches both hosts +through the front door's port-forward, as `*.localhost` resolves to loopback. + The test issuer is nginx serving a discovery document and a JWKS. The suite holds its signing key (`.e2e/issuer-signing.key`), so it can mint tokens with claims Dex never issues. Use it for claims; use Dex for anything a real login would do. Dex has two demo users, `alice@example.com` and `bob@example.com` (password `password`), which Kubernetes sees as `oidc:alice@example.com` and -`oidc:bob@example.com`, plus 200 generated rehearsal users. There are two clients: `krm-foyer`, whose tokens the cluster -accepts, and `other-app`, whose tokens it must reject. +`oidc:bob@example.com`, plus 200 generated rehearsal users. There are three clients: +`krm-foyer`, whose tokens the cluster accepts; `kubectl`, the operator's command line, +whose tokens it accepts under another name, `kubectl:alice@example.com`, as a +[browser identity](application-scope.md#a-browser-identity-in-kubernetes) needs; and +`other-app`, whose tokens it must reject. `kubectl` trusts `krm-foyer` as a peer, so +krm-foyer can ask for its audience with Dex's cross-client scope, and the suite checks +that such a token is still the browser's. ```bash task e2e-up # start or reuse the fixture, with Traefik (about a minute the first time) @@ -427,6 +459,13 @@ task test-e2e # run the suite; brings the fixture up and deploys krm-foyer fir task e2e-down # remove the cluster, Dex, the network and the certificates ``` +After the suite replaces krm-foyer's pod, it removes the node's connection tracking that +still leads to the gone pod (`fixture.forgetGonePods`). Otherwise the suite, one client +making thousands of connections a minute, reuses client ports whose stale entries send +its SYNs to an address nobody answers, and the rehearsal's 1,800 connections at once +time out. [rollout-connections.md](investigations/rollout-connections.md) has the +measurements. + When something fails, the API server's view is usually the answer: `docker logs k3d-krm-foyer-e2e-server-0` shows authenticator errors, and `docker exec k3d-krm-foyer-e2e-server-0 cat /etc/krm-foyer-e2e/audit.log` shows who it diff --git a/internal/auth/check.go b/internal/auth/check.go new file mode 100644 index 0000000..5d0b2ee --- /dev/null +++ b/internal/auth/check.go @@ -0,0 +1,197 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "time" + + authenticationv1 "k8s.io/api/authentication/v1" + + "github.com/ConfigButler/krm-foyer/internal/interruption" + "github.com/ConfigButler/krm-foyer/internal/session" +) + +// IdentityHeader carries /auth/check's identity, when the ingress asks for it: the +// base64url encoding (unpadded) of a JSON CheckIdentity. One field, so an ingress +// copies it whole, and a backend parses one value, never a choice between several. +const IdentityHeader = "Krm-Foyer-Identity" + +// The headers an ingress sends /auth/check to say which request it is checking. +// Traefik's ForwardAuth sets both; nginx's auth_request needs them set in the recipe. +const ( + forwardedMethod = "X-Forwarded-Method" + forwardedURI = "X-Forwarded-Uri" + originalURI = "X-Original-Uri" +) + +// CheckIdentity is who /auth/check says the user is, for a domain backend on the same +// origin: the API server's own answer to a SelfSubjectReview with the user's token, +// and the session's display name and connector as /auth/session shows them. It never +// holds a token. +type CheckIdentity struct { + UserInfo authenticationv1.UserInfo `json:"userInfo"` + DisplayName string `json:"displayName"` + Connector string `json:"connector,omitempty"` + Issuer string `json:"issuer"` + ExpiresAt time.Time `json:"expiresAt"` +} + +// Identify answers who Kubernetes takes the user of r to be. It is given the request +// as the ingress forwarded it (the method the browser used, the page it asked for), +// and answers r itself when it cannot say, returning false: the session refused, a +// bound reached, the API server's refusal, or a failure. +type Identify func(w http.ResponseWriter, r *http.Request) (authenticationv1.UserInfo, bool) + +// Check serves GET /auth/check for an ingress: is the browser behind this request +// signed in, and may it make it? See docs/ingress.md, "The check". +// +// - 204 when it may. With ?identity=true, the answer carries IdentityHeader, which +// identify fills. +// - The refusal /k8s would give otherwise: 401 with no session, 403 for a mutation +// without CSRF proof or from another origin, as a Status for code and a page for a +// person. +// - With ?redirect=true, a page load with no session is a 302 to the login instead, +// at the public URL, which returns to the page the ingress names. +// +// The request checked is the one the ingress forwards: its method in +// X-Forwarded-Method, and its page in X-Forwarded-Uri or X-Original-Uri. Neither +// grants anything a browser could not get by asking for itself, so neither is trusted +// for more: a page that is not a local path becomes /. +func (a *Auth) Check(identify Identify) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-store") + q, err := url.ParseQuery(r.URL.RawQuery) + redirect, okRedirect := flag(q, "redirect") + withIdentity, okIdentity := flag(q, "identity") + unknown := false + for key := range q { + unknown = unknown || key != "redirect" && key != "identity" + } + if err != nil || !okRedirect || !okIdentity || unknown { + badCheck.Write(w) + return + } + checked, ok := forwarded(r) + if !ok { + badCheck.Write(w) + return + } + s, err := a.cfg.Sessions.Use(checked) + if err != nil { + if redirect && isPageLoad(checked) && errors.Is(err, session.ErrNoSession) { + // Absolute, on the configured public origin: an ingress resolves a + // relative Location against the check's own address (Traefik does), which + // is krm-foyer's Service, not the browser's origin. Not http.Redirect: it + // would clean the path. localPath allows nothing that could end the header. + w.Header().Set("Location", strings.TrimSuffix(a.redirectURI, "/auth/callback")+ + "/auth/login?"+url.Values{"return_to": {checked.RequestURI}}.Encode()) + w.WriteHeader(http.StatusFound) + return + } + refused := a.refusal(err) + a.logger.Info("refused", "by", "krm-foyer", "status", refused.Status, "reason", refused.Reason, + "method", checked.Method, "route", "/auth/check") + refused.Serve(w, checked) + return + } + if !withIdentity { + w.WriteHeader(http.StatusNoContent) + return + } + info, ok := identify(w, checked) + if !ok { + return + } + var id identity + claims, err := tokenClaims(s.IDToken) + if err == nil { + id, err = a.claims.extract(claims) + } + if err != nil { + // sessionClaims changed since this login: the claims show nothing rather + // than a guess, as in /auth/session. Who Kubernetes says the user is stands. + a.logger.Warn("a session's claims do not match sessionClaims") + id = identity{} + } + body, err := json.Marshal(CheckIdentity{ + UserInfo: info, DisplayName: id.DisplayName, Connector: id.Connector, + Issuer: s.Issuer, ExpiresAt: s.Expires, + }) + if err != nil { + panic(err) // strings, lists and a time: cannot fail + } + w.Header().Set(IdentityHeader, base64.RawURLEncoding.EncodeToString(body)) + w.WriteHeader(http.StatusNoContent) + }) +} + +// badCheck answers a check the ingress asked wrongly: an unknown option, or a +// forwarded method that is not one. +var badCheck = &interruption.Interruption{ + Status: http.StatusBadRequest, Reason: "BadRequest", + Message: "/auth/check takes only redirect=true and identity=true, and one X-Forwarded-Method", +} + +// flag reads an option of /auth/check: absent, or exactly once as true. +func flag(q url.Values, key string) (on, ok bool) { + v, given := q[key] + if !given { + return false, true + } + return true, len(v) == 1 && v[0] == "true" +} + +// forwarded is the request the ingress is checking, as krm-foyer checks its own: the +// browser's method, the page it asked for, and the browser's own headers, Cookie, +// Origin and the CSRF proof among them. With no X-Forwarded-Method, the check is of +// a GET. More than one, or one that is not a method, is no request at all. +func forwarded(r *http.Request) (*http.Request, bool) { + checked := r.Clone(r.Context()) + if methods, given := r.Header[forwardedMethod]; given { + if len(methods) != 1 || !isToken(methods[0]) { + return nil, false + } + checked.Method = methods[0] + } else { + checked.Method = http.MethodGet + } + checked.RequestURI = "/" + for _, name := range []string{forwardedURI, originalURI} { + if v := r.Header.Values(name); len(v) == 1 && localPath(v[0]) { + checked.RequestURI = v[0] + break + } + } + return checked, true +} + +// isPageLoad reports whether the browser is loading a page: a GET or HEAD that is not +// a fetch from a script, as far as the browser says. Only a page load is sent to the +// login; a script gets the 401 it can act on. +func isPageLoad(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + mode := r.Header.Values("Sec-Fetch-Mode") + return len(mode) == 0 || len(mode) == 1 && mode[0] == "navigate" +} + +// isToken reports whether s is an HTTP method: a token of RFC 9110. +func isToken(s string) bool { + if s == "" { + return false + } + for i := range len(s) { + c := s[i] + if c <= ' ' || c > '~' || c == '"' || c == '(' || c == ')' || c == ',' || c == '/' || c == ':' || + c == ';' || c == '<' || c == '=' || c == '>' || c == '?' || c == '@' || c == '[' || c == '\\' || + c == ']' || c == '{' || c == '}' { + return false + } + } + return true +} diff --git a/internal/auth/check_test.go b/internal/auth/check_test.go new file mode 100644 index 0000000..86ea425 --- /dev/null +++ b/internal/auth/check_test.go @@ -0,0 +1,287 @@ +package auth + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + "sync" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + + "github.com/ConfigButler/krm-foyer/internal/interruption" +) + +// fakeIdentify stands in for the API server's SelfSubjectReview: it records the +// request it was asked about, and answers as the API server would. +type fakeIdentify struct { + mu sync.Mutex + asked []*http.Request + refuse bool +} + +var aliceInfo = authenticationv1.UserInfo{ + Username: "oidc:alice@example.com", UID: "uid-alice", + Groups: []string{"system:authenticated", "oidc:voters"}, + Extra: map[string]authenticationv1.ExtraValue{"configbutler.ai/claims/email": {alice}}, +} + +func (f *fakeIdentify) identify(w http.ResponseWriter, r *http.Request) (authenticationv1.UserInfo, bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.asked = append(f.asked, r) + if f.refuse { + (&interruption.Interruption{Status: http.StatusServiceUnavailable, Reason: "ServiceUnavailable", Message: "no answer"}).Serve(w, r) + return authenticationv1.UserInfo{}, false + } + return aliceInfo, true +} + +func (f *fakeIdentify) calls() []*http.Request { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*http.Request(nil), f.asked...) +} + +// checkHarness serves /auth/check beside the login routes, as the binary does. +func checkHarness(t *testing.T, login LoginConfig) (*harness, *fakeIdentify) { + t.Helper() + h := newHarnessWith(t, login) + f := &fakeIdentify{} + mux := http.NewServeMux() + mux.Handle("/auth/", h.auth.Handler()) + mux.Handle("GET /auth/check", h.auth.Check(f.identify)) + var handler http.Handler = mux + h.handler.Store(&handler) + return h, f +} + +func decodeIdentity(t *testing.T, header http.Header) CheckIdentity { + t.Helper() + values := header.Values(IdentityHeader) + if len(values) != 1 { + t.Fatalf("%s = %q, want one value", IdentityHeader, values) + } + raw, err := base64.RawURLEncoding.DecodeString(values[0]) + if err != nil { + t.Fatal(err) + } + var id CheckIdentity + if err := json.Unmarshal(raw, &id); err != nil { + t.Fatal(err) + } + return id +} + +// The forged copy a browser may send of the identity header, which must never come +// back as if krm-foyer had said it. +var forgedIdentity = base64.RawURLEncoding.EncodeToString([]byte(`{"userInfo":{"username":"system:admin"},"connector":"room-pass"}`)) + +// Signed out, the check answers as /k8s does: a 401 Status for code, a page with a +// sign-in link back to the page the ingress names for a person, and, when the ingress +// asks, a redirect to the login for a page load only. It never names an identity, +// whatever the browser sent, and asks no one who the user is. +func TestCheckSignedOut(t *testing.T) { + h, f := checkHarness(t, LoginConfig{}) + b := h.browser() + forged := http.Header{IdentityHeader: {forgedIdentity}} + + code := b.do(http.MethodGet, "/auth/check?identity=true", forged) + if code.code != http.StatusUnauthorized || !hasReason(code.body, "Unauthorized") || + code.header.Get(interruption.Header) != "Unauthorized" || code.header.Get("Cache-Control") != "no-store" { + t.Errorf("a fetch: %d %v %s", code.code, code.header, code.body) + } + + page := b.do(http.MethodGet, "/auth/check", http.Header{ + "Sec-Fetch-Dest": {"document"}, "Sec-Fetch-Mode": {"navigate"}, "X-Forwarded-Uri": {"/room?id=7"}, + }) + if page.code != http.StatusUnauthorized || !strings.Contains(page.body, `/auth/login?return_to=%2Froom%3Fid%3D7`) { + t.Errorf("a page load: %d %s", page.code, page.body) + } + + for name, tc := range map[string]struct { + header http.Header + location string + }{ + "Traefik's page": {http.Header{"X-Forwarded-Uri": {"/room?id=7"}}, "/auth/login?return_to=%2Froom%3Fid%3D7"}, + "nginx's page": {http.Header{"X-Original-Uri": {"/coffee"}}, "/auth/login?return_to=%2Fcoffee"}, + "a navigation": {http.Header{"Sec-Fetch-Mode": {"navigate"}, "X-Forwarded-Uri": {"/a"}}, "/auth/login?return_to=%2Fa"}, + "no page": {nil, "/auth/login?return_to=%2F"}, + "another site": {http.Header{"X-Forwarded-Uri": {"//evil.example/"}}, "/auth/login?return_to=%2F"}, + "a backslash": {http.Header{"X-Forwarded-Uri": {`/\evil.example`}}, "/auth/login?return_to=%2F"}, + "an absolute URL": {http.Header{"X-Forwarded-Uri": {"https://evil.example/"}}, "/auth/login?return_to=%2F"}, + "two pages": {http.Header{"X-Forwarded-Uri": {"/a", "/b"}}, "/auth/login?return_to=%2F"}, + "a forged one, then nginx's": {http.Header{"X-Forwarded-Uri": {"//evil.example"}, "X-Original-Uri": {"/b"}}, "/auth/login?return_to=%2Fb"}, + } { + got := b.do(http.MethodGet, "/auth/check?redirect=true", tc.header) + // Absolute, at the public URL: an ingress resolves a relative Location against + // the check's own address. + if want := h.foyer.URL + tc.location; got.code != http.StatusFound || got.header.Get("Location") != want || got.header.Get(IdentityHeader) != "" { + t.Errorf("%s: %d to %q, want 302 to %q", name, got.code, got.header.Get("Location"), want) + } + } + + // A script gets the 401 it can act on, and a write is never sent to a login page. + for name, header := range map[string]http.Header{ + "a fetch": {"Sec-Fetch-Mode": {"cors"}}, + "a POST": {"X-Forwarded-Method": {http.MethodPost}}, + } { + got := b.do(http.MethodGet, "/auth/check?redirect=true", header) + if got.code != http.StatusUnauthorized || !hasReason(got.body, "Unauthorized") { + t.Errorf("%s: %d %s, want the 401 Status", name, got.code, got.body) + } + } + for _, resp := range []response{code, page} { + if resp.header.Get(IdentityHeader) != "" { + t.Errorf("a signed-out answer carries %s", IdentityHeader) + } + } + if n := len(f.calls()); n != 0 { + t.Errorf("asked who a signed-out browser is %d times", n) + } +} + +// Signed in, the gate's answer is 204 and nothing more; with identity=true it is the +// API server's answer, with the session's display name and connector, in one header. +// The identity is asked about the request the ingress forwards, and a browser's own +// copy of the header is never what comes back. +func TestCheckSignedIn(t *testing.T) { + login := LoginConfig{SessionClaims: SessionClaims{Connector: "/federated_claims/connector_id"}} + h, f := checkHarness(t, login) + h.issuer.mu.Lock() + h.issuer.tamper = func(claims map[string]any) { + claims["name"] = "Alice Example" + claims["federated_claims"] = map[string]any{"connector_id": "room-pass"} + } + h.issuer.mu.Unlock() + b := h.browser() + b.login(alice, "/") + _, s := b.session() + + gate := b.do(http.MethodGet, "/auth/check", http.Header{IdentityHeader: {forgedIdentity}}) + if gate.code != http.StatusNoContent || gate.header.Get(IdentityHeader) != "" || gate.header.Get("Cache-Control") != "no-store" { + t.Errorf("the page gate: %d %v", gate.code, gate.header) + } + if n := len(f.calls()); n != 0 { + t.Errorf("the page gate asked who the user is %d times", n) + } + + got := b.do(http.MethodGet, "/auth/check?identity=true", http.Header{ + IdentityHeader: {forgedIdentity}, "X-Forwarded-Uri": {"/public/vote"}, + }) + if got.code != http.StatusNoContent { + t.Fatalf("identity: %d %s", got.code, got.body) + } + id := decodeIdentity(t, got.header) + if id.UserInfo.Username != aliceInfo.Username || id.UserInfo.UID != aliceInfo.UID || + strings.Join(id.UserInfo.Groups, ",") != "system:authenticated,oidc:voters" || + id.UserInfo.Extra["configbutler.ai/claims/email"][0] != alice || + id.DisplayName != "Alice Example" || id.Connector != "room-pass" || + id.Issuer != h.issuer.URL || !id.ExpiresAt.Equal(*s.ExpiresAt) { + t.Errorf("identity = %+v", id) + } + raw, _ := base64.RawURLEncoding.DecodeString(got.header.Get(IdentityHeader)) + if strings.Contains(string(raw), h.issuer.idTokens[0]) || strings.Contains(string(raw), "system:admin") { + t.Errorf("the identity holds a token or the browser's forgery: %s", raw) + } + asked := f.calls() + if len(asked) != 1 || asked[0].Method != http.MethodGet || asked[0].RequestURI != "/public/vote" { + t.Fatalf("asked about %v", asked) + } +} + +// A write to a domain backend needs what a write to /k8s needs: the session's CSRF +// proof, from krm-foyer's origin. The ingress names the browser's method; without +// the proof the answer is the 403 /k8s would give, and no one is asked who the user is. +func TestCheckHoldsWritesToTheCSRFRules(t *testing.T) { + h, f := checkHarness(t, LoginConfig{}) + b := h.browser() + b.login(alice, "/") + _, s := b.session() + post := func(header http.Header) response { + header.Set("X-Forwarded-Method", http.MethodPost) + return b.do(http.MethodGet, "/auth/check?identity=true", header) + } + + for name, tc := range map[string]struct { + header http.Header + reason string + }{ + "no proof": {http.Header{"Origin": {h.foyer.URL}}, "CSRFProofRequired"}, + "another site's": {http.Header{"Origin": {"https://evil.example"}, csrfKey: {s.CSRFToken}}, "CrossOriginRequest"}, + "a wrong token": {http.Header{"Origin": {h.foyer.URL}, csrfKey: {"guess"}}, "CSRFProofRequired"}, + "no origin, no fetch": {http.Header{csrfKey: {s.CSRFToken}}, "CrossOriginRequest"}, + "a cross-site fetch": {http.Header{"Sec-Fetch-Site": {"same-site"}, csrfKey: {s.CSRFToken}}, "CrossOriginRequest"}, + "two tokens, one right": {http.Header{"Origin": {h.foyer.URL}, csrfKey: {s.CSRFToken, "x"}}, "CSRFProofRequired"}, + } { + got := post(tc.header) + if got.code != http.StatusForbidden || !hasReason(got.body, tc.reason) || got.header.Get(IdentityHeader) != "" { + t.Errorf("%s: %d %s, want 403 %s", name, got.code, got.body, tc.reason) + } + } + if n := len(f.calls()); n != 0 { + t.Fatalf("asked who the user is for %d refused writes", n) + } + + // A method spelt any other way is a write too, not a read. + got := b.do(http.MethodGet, "/auth/check?identity=true", http.Header{"X-Forwarded-Method": {"get"}}) + if got.code != http.StatusForbidden { + t.Errorf("a lower-case get: %d, want the 403 of a write", got.code) + } + + ok := post(http.Header{"Origin": {h.foyer.URL}, csrfKey: {s.CSRFToken}}) + if ok.code != http.StatusNoContent || decodeIdentity(t, ok.header).UserInfo.Username != aliceInfo.Username { + t.Errorf("a write with proof: %d %v", ok.code, ok.header) + } + if asked := f.calls(); len(asked) != 1 || asked[0].Method != http.MethodPost { + t.Errorf("asked about %v, want the POST", asked) + } +} + +// When the API server cannot say who the user is, its answer is the check's, with no +// identity: never one guessed from the token. +func TestCheckWithoutAnAnswer(t *testing.T) { + h, f := checkHarness(t, LoginConfig{}) + f.refuse = true + b := h.browser() + b.login(alice, "/") + got := b.do(http.MethodGet, "/auth/check?identity=true", nil) + if got.code != http.StatusServiceUnavailable || got.header.Get(IdentityHeader) != "" { + t.Errorf("%d %v", got.code, got.header) + } +} + +// An ingress that asks wrongly is told so, rather than given a guess at what it meant: +// identity=1 must not quietly be a check without identity. +func TestCheckOptions(t *testing.T) { + h, f := checkHarness(t, LoginConfig{}) + b := h.browser() + b.login(alice, "/") + for name, tc := range map[string]struct { + query string + header http.Header + }{ + "identity=1": {"identity=1", nil}, + "identity twice": {"identity=true&identity=true", nil}, + "redirect=yes": {"redirect=yes", nil}, + "an unknown option": {"identity=true&user=admin", nil}, + "a broken query": {"identity=%zz", nil}, + "two methods": {"", http.Header{"X-Forwarded-Method": {"GET", "POST"}}}, + "an empty method": {"", http.Header{"X-Forwarded-Method": {""}}}, + "a method with a space": {"", http.Header{"X-Forwarded-Method": {"GET POST"}}}, + } { + got := b.do(http.MethodGet, "/auth/check?"+tc.query, tc.header) + if got.code != http.StatusBadRequest || !hasReason(got.body, "BadRequest") || got.header.Get(IdentityHeader) != "" { + t.Errorf("%s: %d %s, want 400", name, got.code, got.body) + } + } + if n := len(f.calls()); n != 0 { + t.Errorf("asked who the user is for %d bad checks", n) + } + for _, query := range []string{"", "identity=true", "redirect=true", "redirect=true&identity=true"} { + if got := b.do(http.MethodGet, "/auth/check?"+query, nil); got.code != http.StatusNoContent { + t.Errorf("%q: %d %s", query, got.code, got.body) + } + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 3a939eb..bca9ce0 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,8 @@ type Config struct { Auth http.Handler // WhoAmI serves GET /auth/whoami. Nil leaves it to Auth. WhoAmI http.Handler + // Check serves GET /auth/check, for an ingress. Nil leaves it to Auth. + Check http.Handler // Ready reports whether krm-foyer can serve logins yet. Nil means always. Ready func() bool } @@ -47,6 +49,9 @@ func New(cfg Config) http.Handler { if cfg.WhoAmI != nil { mux.Handle("GET /auth/whoami", cfg.WhoAmI) } + if cfg.Check != nil { + mux.Handle("GET /auth/check", cfg.Check) + } if cfg.Stream != nil { // Every method, so that the stream answers one that is not a GET with an // interruption of its own rather than ServeMux's plain 405. diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 63e7bde..dfb03a6 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -182,6 +182,28 @@ func TestWhoAmIRoute(t *testing.T) { } } +// GET /auth/check, with its query, reaches the check; any other method goes to Auth, +// which has no such route. +func TestCheckRoute(t *testing.T) { + var reached []string + record := func(who string) http.Handler { + return http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + reached = append(reached, who+" "+r.Method+" "+r.URL.RequestURI()) + }) + } + h := New(Config{Version: "test", Auth: record("auth"), WhoAmI: record("whoami"), Check: record("check")}) + for _, r := range []struct{ method, target string }{ + {http.MethodGet, "/auth/check?identity=true"}, {http.MethodHead, "/auth/check"}, + {http.MethodPost, "/auth/check"}, {http.MethodGet, "/auth/check/x"}, + } { + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequestWithContext(t.Context(), r.method, r.target, nil)) + } + want := "check GET /auth/check?identity=true, check HEAD /auth/check, auth POST /auth/check, auth GET /auth/check/x" + if got := strings.Join(reached, ", "); got != want { + t.Errorf("reached %s, want %s", got, want) + } +} + // /stream/v1 reaches the stream with any method, so that it answers the ones it // refuses itself; no other path under /stream does. func TestStreamRoute(t *testing.T) { diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 08d9a42..427908f 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -61,6 +61,8 @@ type Streams struct { writes time.Duration // shared is nil without shared watches. shared *shared + // identities are /auth/check's reused answers to who a token belongs to. + identities *identities } // New returns streams from cfg.Server. @@ -73,7 +75,8 @@ func New(cfg Config) (*Streams, error) { return nil, errors.New("no gate") } s := &Streams{ - server: url.URL{Scheme: cfg.Server.Scheme, Host: cfg.Server.Host}, + identities: newIdentities(nil), + server: url.URL{Scheme: cfg.Server.Scheme, Host: cfg.Server.Host}, // The same pinned transport as the proxy's. The user's token is added per // request by client-go, from the rest.Config built for that user alone. transport: &http.Transport{ diff --git a/internal/stream/whoami.go b/internal/stream/whoami.go index a71d9a3..3f73c60 100644 --- a/internal/stream/whoami.go +++ b/internal/stream/whoami.go @@ -1,9 +1,12 @@ package stream import ( + "crypto/sha256" "encoding/json" "errors" "net/http" + "strings" + "sync" "time" authenticationv1 "k8s.io/api/authentication/v1" @@ -30,37 +33,141 @@ type whoami struct { // other credential to try. func (s *Streams) WhoAmI() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - a := s.gate.Admit(w, r) - if a == nil { - return - } - defer a.Close() - u := &upstream{streams: s, userAgent: r.UserAgent()} - info, err := u.selfSubjectReview(a.Request.Context(), a.Credential) - var api apierrors.APIStatus - switch { - case err == nil && info.Username != "": - writeWhoami(w, a.Credential, info) - case err == nil: - s.logger.Warn("the API server named no user for /auth/whoami") - a.Interrupt(w, unknownIdentity) - case errors.As(err, &api) && api.Status().Code >= 400 && api.Status().Code < 500: - // The API server's own refusal, as it gave it: a token it does not accept - // is its 401, not krm-foyer's. - a.Refused(gate.ByKubernetes, "status", api.Status().Code, "reason", string(api.Status().Reason)) - st := api.Status() - body, _ := json.Marshal(&st) - w.Header().Set("Content-Type", "application/json") - w.Header().Set("Cache-Control", "no-store") - w.WriteHeader(int(st.Code)) - _, _ = w.Write(body) - default: - s.logger.Warn("could not find who the user is for /auth/whoami", "cause", class(err)) - a.Interrupt(w, unknownIdentity) + info, cred, ok := s.identify(w, r, nil) + if ok { + writeWhoami(w, cred, info) } }) } +// Identify is /auth/check's question, asked as /auth/whoami asks it, through the gate +// with the user's own token, and answered on failure as /auth/whoami answers. One +// difference: an answer is reused for IdentityTTL, as a domain backend may check on +// every request. +func (s *Streams) Identify(w http.ResponseWriter, r *http.Request) (authenticationv1.UserInfo, bool) { + info, _, ok := s.identify(w, r, s.identities) + return info, ok +} + +// identify asks the API server who r's user is, or answers r itself and returns false. +// With reuse, a recent answer for the same token stands in for the review. +func (s *Streams) identify(w http.ResponseWriter, r *http.Request, reuse *identities) (authenticationv1.UserInfo, gate.Credential, bool) { + a := s.gate.Admit(w, r) + if a == nil { + return authenticationv1.UserInfo{}, gate.Credential{}, false + } + defer a.Close() + if info, ok := reuse.get(a.Token); ok { + return info, a.Credential, true + } + u := &upstream{streams: s, userAgent: r.UserAgent()} + info, err := u.selfSubjectReview(a.Request.Context(), a.Credential) + var api apierrors.APIStatus + switch { + case err == nil && info.Username != "": + reuse.put(a.Token, info, a.Expires) + return info, a.Credential, true + case err == nil: + s.logger.Warn("the API server named no user", "route", route(r)) + a.Interrupt(w, unknownIdentity) + case errors.As(err, &api) && api.Status().Code >= 400 && api.Status().Code < 500: + // The API server's own refusal, as it gave it: a token it does not accept + // is its 401, not krm-foyer's. + a.Refused(gate.ByKubernetes, "status", api.Status().Code, "reason", string(api.Status().Reason)) + st := api.Status() + body, _ := json.Marshal(&st) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(int(st.Code)) + _, _ = w.Write(body) + default: + s.logger.Warn("could not find who the user is", "route", route(r), "cause", class(err)) + a.Interrupt(w, unknownIdentity) + } + return authenticationv1.UserInfo{}, gate.Credential{}, false +} + +// route names the route r arrived on, for a log line. +func route(r *http.Request) string { + if strings.HasPrefix(r.URL.Path, "/auth/check") { + return "/auth/check" + } + return "/auth/whoami" +} + +// IdentityTTL is how long /auth/check reuses the API server's answer to who a token +// belongs to. The answer depends only on the token and the cluster's authentication +// configuration, so this is how long a change to that configuration may take to reach +// a domain backend; access decisions are not reused here, and are never krm-foyer's. +const IdentityTTL = 30 * time.Second + +// maxIdentities bounds the answers kept: one per signed-in token, beyond which a check +// asks the API server again rather than keep more. +const maxIdentities = 10000 + +// identities reuses SelfSubjectReview answers by token, for at most IdentityTTL and +// never past the session's end. The key is the token's SHA-256, so no token is kept. +// A nil *identities reuses nothing. +type identities struct { + ttl time.Duration + max int + now func() time.Time + + mu sync.Mutex + entries map[[sha256.Size]byte]knownIdentity +} + +type knownIdentity struct { + info authenticationv1.UserInfo + until time.Time +} + +func newIdentities(now func() time.Time) *identities { + if now == nil { + now = time.Now + } + return &identities{ttl: IdentityTTL, max: maxIdentities, now: now, entries: map[[sha256.Size]byte]knownIdentity{}} +} + +func (c *identities) get(token string) (authenticationv1.UserInfo, bool) { + if c == nil { + return authenticationv1.UserInfo{}, false + } + key := sha256.Sum256([]byte(token)) + c.mu.Lock() + defer c.mu.Unlock() + e, ok := c.entries[key] + if !ok || !c.now().Before(e.until) { + delete(c.entries, key) + return authenticationv1.UserInfo{}, false + } + return *e.info.DeepCopy(), true +} + +func (c *identities) put(token string, info authenticationv1.UserInfo, sessionEnd time.Time) { + if c == nil { + return + } + now := c.now() + until := now.Add(c.ttl) + if sessionEnd.Before(until) { + until = sessionEnd + } + c.mu.Lock() + defer c.mu.Unlock() + if len(c.entries) >= c.max { + for k, e := range c.entries { + if !now.Before(e.until) { + delete(c.entries, k) + } + } + if len(c.entries) >= c.max { + return + } + } + c.entries[sha256.Sum256([]byte(token))] = knownIdentity{info: *info.DeepCopy(), until: until} +} + // unknownIdentity is the answer when the API server could not say who the user is. var unknownIdentity = &interruption.Interruption{ Status: http.StatusServiceUnavailable, Reason: "ServiceUnavailable", diff --git a/internal/stream/whoami_test.go b/internal/stream/whoami_test.go index b73babd..b19b1c7 100644 --- a/internal/stream/whoami_test.go +++ b/internal/stream/whoami_test.go @@ -2,6 +2,7 @@ package stream import ( "context" + "crypto/sha256" "crypto/x509" "encoding/json" "io" @@ -9,10 +10,13 @@ import ( "net/http/httptest" "net/url" "strings" + "sync" "sync/atomic" "testing" "time" + authenticationv1 "k8s.io/api/authentication/v1" + "github.com/ConfigButler/krm-foyer/internal/gate" "github.com/ConfigButler/krm-foyer/internal/interruption" ) @@ -163,3 +167,153 @@ func TestWhoAmIFailsClosed(t *testing.T) { t.Fatalf("the redirect was followed %d times", n) } } + +// identifyFoyer serves Identify from api behind a gate with creds, answering 204 when +// it names the user, with the identities' clock at *now. +func identifyFoyer(t *testing.T, api *apiServer, creds gate.Credentials, now *time.Time) (*Streams, *httptest.Server) { + t.Helper() + g, err := gate.New(gate.Config{Credentials: creds}) + if err != nil { + t.Fatal(err) + } + server, _ := url.Parse(api.URL) + roots := x509.NewCertPool() + roots.AddCert(api.Certificate()) + s, err := New(Config{Server: server, RootCAs: roots, Gate: g}) + if err != nil { + t.Fatal(err) + } + s.identities = newIdentities(func() time.Time { return *now }) + front := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if info, ok := s.Identify(w, r); ok { + w.Header().Set("X-User", info.Username) + w.WriteHeader(http.StatusNoContent) + } + })) + t.Cleanup(front.Close) + return s, front +} + +// /auth/check may be asked on every request to a domain backend. One review answers +// a token's checks for IdentityTTL, never past the session's end, and never for +// another token; a failure is not kept, and the next check asks again. +func TestIdentifyReusesAnAnswer(t *testing.T) { + now := sessionEnd.Add(-time.Hour) + api := newSharedAPI(t) + creds := &switchable{token: userToken} + _, front := identifyFoyer(t, api.apiServer, creds, &now) + reviews := func() int { + n := 0 + for _, r := range api.received() { + if strings.HasSuffix(r.URL.Path, "/selfsubjectreviews") { + n++ + } + } + return n + } + check := func() (int, string) { + code, header, _ := getWhoami(t, front, nil) + return code, header.Get("X-User") + } + + for range 3 { + if code, user := check(); code != http.StatusNoContent || user != "oidc:alice@example.com" { + t.Fatalf("%d %q", code, user) + } + } + if n := reviews(); n != 1 { + t.Fatalf("%d reviews for three checks of one token, want 1", n) + } + + creds.set(otherToken) + if _, user := check(); user != "oidc:bob@example.com" { + t.Fatalf("another token was answered as %q", user) + } + if n := reviews(); n != 2 { + t.Fatalf("%d reviews, want another token to be asked about", n) + } + + creds.set(userToken) + now = now.Add(IdentityTTL) + check() + if n := reviews(); n != 3 { + t.Fatalf("%d reviews, want the answer to expire after IdentityTTL", n) + } + + // Within IdentityTTL of the session's end, the answer lasts only until that end. + now = sessionEnd.Add(-IdentityTTL / 2) + check() + now = sessionEnd.Add(-time.Nanosecond) + check() + if n := reviews(); n != 4 { + t.Fatalf("%d reviews, want one answer to the end of the session", n) + } + + api.ssr = status(http.StatusInternalServerError, "InternalError", "etcd") + now = now.Add(-time.Minute) // a new answer is needed: the last expired at the session's end + creds.set(otherToken) // its answer expired long ago + for range 2 { + if code, _ := check(); code != http.StatusServiceUnavailable { + t.Fatalf("a failing API server: %d", code) + } + } + if n := reviews(); n != 6 { + t.Fatalf("%d reviews, want a failure to be asked again, not kept", n) + } +} + +// The answers kept are bounded: past the bound, a check asks the API server rather +// than keep one more, and expired answers make room. +func TestIdentitiesAreBounded(t *testing.T) { + now := time.Unix(1_000_000, 0) + c := newIdentities(func() time.Time { return now }) + c.max = 2 + end := now.Add(time.Hour) + c.put("a", authenticationv1.UserInfo{Username: "a"}, end) + c.put("b", authenticationv1.UserInfo{Username: "b"}, end) + c.put("c", authenticationv1.UserInfo{Username: "c"}, end) + if _, ok := c.get("c"); ok { + t.Fatal("kept an answer past the bound") + } + if info, ok := c.get("a"); !ok || info.Username != "a" { + t.Fatal("lost an answer within the bound") + } + now = now.Add(IdentityTTL) + c.put("c", authenticationv1.UserInfo{Username: "c"}, end) + if info, ok := c.get("c"); !ok || info.Username != "c" { + t.Fatal("expired answers did not make room") + } + if len(c.entries) != 1 { + t.Fatalf("%d answers kept, want the expired ones gone", len(c.entries)) + } + if _, kept := c.entries[sha256.Sum256([]byte("c"))]; !kept { + t.Fatal("an answer is kept under something other than its token's hash") + } + + // A copy goes out: what a caller does to it changes nothing kept. + c.put("d", authenticationv1.UserInfo{Username: "d", Groups: []string{"g"}}, end) + got, _ := c.get("d") + got.Groups[0] = "system:masters" + if again, _ := c.get("d"); again.Groups[0] != "g" { + t.Fatal("a caller changed a kept answer") + } +} + +// switchable is a credential whose token a test can change. +type switchable struct { + mu sync.Mutex + token string +} + +func (c *switchable) set(token string) { + c.mu.Lock() + defer c.mu.Unlock() + c.token = token +} + +func (c *switchable) Token(*http.Request) (gate.Credential, *interruption.Interruption) { + c.mu.Lock() + defer c.mu.Unlock() + return gate.Credential{Token: c.token, User: "u", Session: c.token, Expires: sessionEnd, + Live: func(context.Context) bool { return true }}, nil +} diff --git a/test/e2e/browser_test.go b/test/e2e/browser_test.go index 4e0ca7c..700cb18 100644 --- a/test/e2e/browser_test.go +++ b/test/e2e/browser_test.go @@ -41,6 +41,13 @@ func (f *fixture) browser() *browser { return f.browserVia(f.client.Transport) } // briefBrowser reaches the brief krm-foyer, whose sessions end within a minute. func (f *fixture) briefBrowser() *browser { return f.browserVia(f.briefTransport) } +// frontDoorBrowser reaches the origin through Traefik, with the routes behind +// /auth/check (traefik-routes.yaml) as well as krm-foyer's own. +func (f *fixture) frontDoorBrowser() *browser { return f.browserVia(f.frontDoorTransport) } + +// nginxBrowser reaches the origin through docs/ingress.md's nginx recipe. +func (f *fixture) nginxBrowser() *browser { return f.browserVia(f.nginxTransport) } + func (f *fixture) browserVia(transport http.RoundTripper) *browser { jar, err := cookiejar.New(nil) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/cluster/authentication-config.yaml b/test/e2e/cluster/authentication-config.yaml index b6007be..36007b2 100644 --- a/test/e2e/cluster/authentication-config.yaml +++ b/test/e2e/cluster/authentication-config.yaml @@ -8,19 +8,39 @@ kind: AuthenticationConfiguration jwt: - issuer: url: https://dex.localhost:5556 - # Only tokens minted for krm-foyer. A token Dex issued to any other client is - # rejected with 401, which the suite checks: an issuer the cluster trusts is not - # the same as every token from that issuer. - audiences: [krm-foyer] + # Only tokens minted for krm-foyer, or for the operator's command line. A token + # Dex issued to any other client is rejected with 401, which the suite checks: an + # issuer the cluster trusts is not the same as every token from that issuer. + audiences: [krm-foyer, kubectl] + audienceMatchPolicy: MatchAny certificateAuthority: | CA_PEM claimMappings: &claimMappings + # A browser identity (docs/application-scope.md): one person, two usernames, + # told apart by the client the issuer gave the token to. Through krm-foyer they + # are oidc:; on the command line kubectl:, which is where an + # operator's broad grants belong. Every script on an application's origin can + # use what krm-foyer's token may do, and with this it may do only what + # oidc: is granted. + # + # The client is azp when the token has one, and azp must then be a string and + # one of its audiences (Dex sets it when a client asks for another audience with + # the cross-client scope). Without azp it is the audience: a string, or a list + # of exactly one. Two audiences and no azp name no client. Each client the + # cluster knows is listed with its prefix, and nothing falls back to one: an + # unknown or doubtful client is no key of the map, the lookup fails, and the + # token is refused. The rule under claimValidationRules says so first, with a + # message. CEL has no variables here, so the client is spelled out each time. username: - claim: email - prefix: "oidc:" + expression: >- + {'krm-foyer': 'oidc:', 'kubectl': 'kubectl:'}[(has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : ''))] + + claims.email + # Groups follow the same split, so a group grant to one is never one to the other. groups: - claim: groups - prefix: "oidc:" + expression: >- + has(claims.groups) ? dyn(claims.groups).map(g, + {'krm-foyer': 'oidc:', 'kubectl': 'kubectl:'}[(has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : ''))] + + g) : [] # The two keys gitops-reverser reads from user.extra, in audit events and # admission requests, for a commit's author (docs/design.md, "Attribution"): # from the signed token, here and nowhere else. A claim that is missing maps to @@ -31,6 +51,11 @@ CA_PEM - key: configbutler.ai/claims/email valueExpression: "claims.?email.orValue('')" claimValidationRules: &claimValidationRules + # The client, as the username above reads it, must be one the cluster names. + - expression: >- + (has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : '')) + in ['krm-foyer', 'kubectl'] + message: "the token's client (azp, or its only audience) must be krm-foyer or kubectl" # Mapping the username from email makes the API server check email_verified on # its own, but that check is claims.?email_verified.orValue(true): a token without # the claim passes. The name is only an identity if the issuer vouched for it, so @@ -47,9 +72,55 @@ CA_PEM # of a copy. - issuer: url: https://issuer.krm-foyer.test:8443 - audiences: [krm-foyer] + # Dex's audiences, so the suite can mint what either client would receive. + audiences: [krm-foyer, kubectl] + audienceMatchPolicy: MatchAny certificateAuthority: | CA_PEM claimMappings: *claimMappings claimValidationRules: *claimValidationRules userValidationRules: *userValidationRules + + # Room Pass's Dex (test/e2e/cluster/room-pass.sh), for its QR login through + # krm-foyer-room. Room Pass's own reference rules (its + # deploy/apiserver/authentication-config.yaml): a participant is the connector + # room-pass, named by Dex's opaque subject, never the name they typed, and may only + # be in demo: groups, so a forged identity, if every control in front of Dex failed, + # would still be in none of the groups the platform binds. The API server reaches the + # issuer through Traefik, at room-pass.localhost's fixed address. + - issuer: + url: https://room-pass.localhost:8443 + audiences: [krm-foyer] + certificateAuthority: | +CA_PEM + claimValidationRules: + # The client, by the same rules as for Dex above: krm-foyer is the only one. + - expression: >- + (has(claims.azp) ? (type(claims.azp) == string && claims.azp in (type(claims.aud) == string ? [claims.aud] : claims.aud) ? claims.azp : '') : (type(claims.aud) == string ? claims.aud : size(claims.aud) == 1 ? claims.aud[0] : '')) + == 'krm-foyer' + message: "the token's client (azp, or its only audience) must be krm-foyer" + # Dex sets federated_claims only for the federated:id scope. + - expression: "has(claims.federated_claims) && claims.federated_claims.connector_id == 'room-pass'" + message: "unknown or missing Dex connector; the client must request the federated:id scope" + - expression: "has(claims.groups) && dyn(claims.groups).all(g, g.startsWith('demo:'))" + message: "the room-pass connector may only assert demo groups" + - expression: "has(claims.email_verified) && type(claims.email_verified) == bool && claims.email_verified" + message: "email_verified must be the boolean true" + claimMappings: + username: + expression: "'demo:' + claims.sub" + groups: + claim: groups + prefix: "" + # The attribution keys, as for Dex above: the name the participant chose, as + # Room Pass stored it, and the address it made from it. + extra: + - key: configbutler.ai/claims/display-name + valueExpression: "claims.?name.orValue('')" + - key: configbutler.ai/claims/email + valueExpression: "claims.?email.orValue('')" + userValidationRules: + - expression: "user.username.startsWith('demo:')" + message: "username must carry the demo: prefix" + - expression: "user.groups.all(group, !group.startsWith('system:'))" + message: "groups must not use the Kubernetes system: prefix" diff --git a/test/e2e/cluster/deploy-foyer.sh b/test/e2e/cluster/deploy-foyer.sh index 8c69b2f..b1ee88e 100755 --- a/test/e2e/cluster/deploy-foyer.sh +++ b/test/e2e/cluster/deploy-foyer.sh @@ -14,12 +14,17 @@ set -euo pipefail : "${IMAGE:?set IMAGE to the krm-foyer image to deploy}" CLUSTER_NAME="${CLUSTER_NAME:-krm-foyer-e2e}" FOYER_HOST="foyer.localhost" +# Its Service's name, for an ingress that calls krm-foyer itself: Traefik's ForwardAuth +# to /auth/check (traefik-routes.yaml) verifies krm-foyer's certificate by this name. +FOYER_SERVICE_HOST="krm-foyer.krm-foyer.svc" # Browsers reach krm-foyer through the front door on this port (front-door.sh), so it is # part of the public URL; the suite goes to the NodePort directly, under the same name. FOYER_URL="https://$FOYER_HOST:8443" NODE_PORT=30443 # A second krm-foyer whose sessions end within a minute (foyer-brief-values.yaml). BRIEF_NODE_PORT=30444 +# docs/ingress.md's nginx recipe in front of the main krm-foyer (front-door.sh). +NGINX_NODE_PORT=30445 here="$(cd "$(dirname "$0")" && pwd)" repo="$(cd "$here/../../.." && pwd)" @@ -42,11 +47,12 @@ mkdir -p "$E2E_DIR/foyer" if [ ! -f "$E2E_DIR/foyer/tls.crt" ] \ || ! openssl x509 -checkend 86400 -noout -in "$E2E_DIR/foyer/tls.crt" >/dev/null 2>&1 \ || ! openssl verify -CAfile "$E2E_DIR/ca.crt" "$E2E_DIR/foyer/tls.crt" >/dev/null 2>&1 \ - || ! openssl x509 -noout -checkhost "$FOYER_HOST" -in "$E2E_DIR/foyer/tls.crt" | grep -q 'does match'; then + || ! openssl x509 -noout -checkhost "$FOYER_HOST" -in "$E2E_DIR/foyer/tls.crt" | grep -q 'does match' \ + || ! openssl x509 -noout -checkhost "$FOYER_SERVICE_HOST" -in "$E2E_DIR/foyer/tls.crt" | grep -q 'does match'; then openssl req -newkey rsa:2048 -nodes -subj "/CN=$FOYER_HOST" \ -keyout "$E2E_DIR/foyer/tls.key" -out "$E2E_DIR/foyer/tls.csr" 2>/dev/null openssl x509 -req -in "$E2E_DIR/foyer/tls.csr" -CA "$E2E_DIR/ca.crt" -CAkey "$E2E_DIR/ca.key" \ - -CAcreateserial -days 30 -extfile <(printf 'subjectAltName=DNS:%s' "$FOYER_HOST") \ + -CAcreateserial -days 30 -extfile <(printf 'subjectAltName=DNS:%s,DNS:%s' "$FOYER_HOST" "$FOYER_SERVICE_HOST") \ -out "$E2E_DIR/foyer/tls.crt" 2>/dev/null fi @@ -109,7 +115,9 @@ cat > "$E2E_DIR/foyer-env" </dev/null +kubectl apply -f "$here/traefik-routes.yaml" >/dev/null + +echo "== the nginx recipe, in front of krm-foyer on a NodePort" +# docs/ingress.md's nginx recipe as the document has it: its nginx block, and its njs +# script, copied out of the document, so a spec of the recipe is a spec of the text. +stage="$E2E_DIR/nginx-door" +mkdir -p "$stage" +recipe() { + awk -v fence='```'"$1" ' + /^\*\*nginx, `auth_request`\.\*\*/ { section = 1 } + section && $0 == fence { inside = 1; next } + inside && $0 == "```" { exit } + inside { print } + ' "$repo/docs/ingress.md" +} +recipe nginx > "$stage/recipe.conf" +recipe js > "$stage/foyer.js" +[ -s "$stage/recipe.conf" ] || { echo "no nginx recipe found in docs/ingress.md" >&2; exit 1; } +apply create configmap nginx-door \ + --from-file=nginx.conf="$here/nginx-door.conf" \ + --from-file=foyer-upstream.conf="$here/nginx-door-upstream.conf" \ + --from-file=recipe.conf="$stage/recipe.conf" --from-file=foyer.js="$stage/foyer.js" \ + --from-file=ca.crt="$E2E_DIR/ca.crt" +config_hash="$(cat "$here"/nginx-door* "$stage"/* "$E2E_DIR/ca.crt" "$E2E_DIR/foyer/tls.crt" | sha256sum | cut -c1-16)" +sed -e "s|NGINX_IMAGE|$NGINX_IMAGE|" -e "s|CONFIG_HASH|$config_hash|" "$here/nginx-door.yaml" \ + | kubectl apply -f - >/dev/null +if ! kubectl -n fixture rollout status deployment/nginx-door --timeout=120s; then + kubectl -n fixture describe pods -l app=nginx-door >&2 + kubectl -n fixture logs deployment/nginx-door --tail=50 >&2 || true + exit 1 +fi + echo "== port-forwards" "$here/port-forward.sh" # Through the forward, as a browser: the example's page, and /auth/ reaching krm-foyer @@ -79,4 +116,20 @@ front() { curl -sS --cacert "$E2E_DIR/ca.crt" --resolve "foyer.localhost:8443:12 front -f https://foyer.localhost:8443/ | grep -q 'Hello, krm-foyer' code="$(front -o /dev/null -w '%{http_code}' https://foyer.localhost:8443/auth/session)" [ "$code" = 401 ] || { echo "the front door does not reach krm-foyer (/auth/session: $code)" >&2; exit 1; } +# Traefik's routes ask krm-foyer first: a signed-out fetch is its 401, a page load its +# redirect to the login. Anything else is a route or a ForwardAuth that does not work. +# Traefik takes a moment to load new routes, and until then the Gateway's catch-all +# answers, so this waits up to 30 seconds for each. +expect_code() { + local path="$1" want="$2" what="$3" code="" + for _ in $(seq 1 30); do + code="$(front -o /dev/null -w '%{http_code}' "https://foyer.localhost:8443$path")" + [ "$code" = "$want" ] && return 0 + sleep 1 + done + echo "$path is not $what (signed out: $code, want $want)" >&2 + exit 1 +} +expect_code /public/whoami 401 "behind /auth/check" +expect_code /members/ 302 "behind the login gate" echo "the hello example is at $FOYER_URL" diff --git a/test/e2e/cluster/hello-web-nginx.conf b/test/e2e/cluster/hello-web-nginx.conf index 2c813ae..181fc38 100644 --- a/test/e2e/cluster/hello-web-nginx.conf +++ b/test/e2e/cluster/hello-web-nginx.conf @@ -39,5 +39,20 @@ http { add_header X-Content-Type-Options nosniff always; add_header Referrer-Policy no-referrer always; add_header Cache-Control no-cache always; + + # A domain backend, as far as the suite needs one (traefik-routes.yaml): it answers + # with the identity header it received, and the method, so a spec sees exactly what + # a backend behind /auth/check?identity=true is told. + location = /public/whoami { + default_type application/json; + add_header Cache-Control no-store always; + return 200 '{"identity":"$http_krm_foyer_identity","method":"$request_method"}'; + } + + # A page behind the login gate (traefik-routes.yaml). + location = /members/ { + default_type text/html; + return 200 'Members

For signed-in members.

'; + } } } diff --git a/test/e2e/cluster/nginx-door-upstream.conf b/test/e2e/cluster/nginx-door-upstream.conf new file mode 100644 index 0000000..c29b69a --- /dev/null +++ b/test/e2e/cluster/nginx-door-upstream.conf @@ -0,0 +1,10 @@ +# How nginx-door.conf reaches krm-foyer: verified TLS to its Service name, the browser's +# Host, and no buffering, so a stream's events pass as they come. +proxy_ssl_verify on; +proxy_ssl_trusted_certificate /etc/nginx/krm-foyer-ca.crt; +proxy_ssl_name krm-foyer.krm-foyer.svc; +proxy_ssl_server_name on; +proxy_http_version 1.1; +proxy_set_header Host $http_host; +proxy_buffering off; +proxy_read_timeout 1h; diff --git a/test/e2e/cluster/nginx-door.conf b/test/e2e/cluster/nginx-door.conf new file mode 100644 index 0000000..4e9b349 --- /dev/null +++ b/test/e2e/cluster/nginx-door.conf @@ -0,0 +1,81 @@ +# The nginx recipe of docs/ingress.md, run: nginx in front of krm-foyer under the same +# public name, foyer.localhost, which the suite reaches on a NodePort (FOYER_NGINX_ADDR in +# foyer-env). front-door.sh copies the recipe's own blocks out of docs/ingress.md into +# recipe.conf and foyer.js, so what runs here is what the document says, byte for byte. +# Only what the recipe leaves to the deployment is written here: TLS, krm-foyer's own +# prefixes, and the two backends it names. + +# The recipe encodes the return path with njs, which the official nginx images include. +load_module modules/ngx_http_js_module.so; + +worker_processes 1; +error_log /dev/stderr warn; +pid /tmp/nginx.pid; + +events {} + +http { + access_log /dev/stdout; + # Writable paths for an unprivileged nginx: /tmp is an emptyDir. + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server_tokens off; + + server { + listen 8443 ssl; + server_name foyer.localhost; + # The front door's certificate (front-door.sh): the same public name as krm-foyer. + ssl_certificate /etc/nginx-door/tls/tls.crt; + ssl_certificate_key /etc/nginx-door/tls/tls.key; + ssl_protocols TLSv1.3; + + # krm-foyer's own prefixes, ungated, to its Service over verified TLS. With no URI + # in proxy_pass, nginx passes the request target as the browser sent it. + location /auth/ { + proxy_pass https://krm-foyer.krm-foyer.svc; + include /etc/nginx-door/foyer-upstream.conf; + } + location /k8s/ { + proxy_pass https://krm-foyer.krm-foyer.svc; + include /etc/nginx-door/foyer-upstream.conf; + } + location /stream/ { + proxy_pass https://krm-foyer.krm-foyer.svc; + include /etc/nginx-door/foyer-upstream.conf; + } + location /_foyer/ { + proxy_pass https://krm-foyer.krm-foyer.svc; + include /etc/nginx-door/foyer-upstream.conf; + } + + # docs/ingress.md's nginx recipe, copied from the document by front-door.sh. + include /etc/nginx/recipe.conf; + + location / { + return 404; + } + } + + # The recipe's two backends, voter-backend and voter-web: the pod's hostAliases send + # both names here. Each answers with what it was sent, so a spec sees exactly what a + # backend behind the recipe receives: the request target, and the identity header. + server { + listen 127.0.0.1:8080; + default_type text/plain; + + # A tripwire, as in hello-web: the recipe removes the session cookie before this hop. + if ($http_cookie) { + return 400; + } + + location /public/ { + return 200 "identity=$http_krm_foyer_identity\nrequest=$request_uri\n"; + } + location /admin/ { + return 200 "request=$request_uri\n"; + } + } +} diff --git a/test/e2e/cluster/nginx-door.yaml b/test/e2e/cluster/nginx-door.yaml new file mode 100644 index 0000000..a7bbb47 --- /dev/null +++ b/test/e2e/cluster/nginx-door.yaml @@ -0,0 +1,92 @@ +# The nginx recipe of docs/ingress.md, run (nginx-door.conf): nginx in front of the main +# krm-foyer, on a NodePort the suite reaches under krm-foyer's public name. front-door.sh +# fills in the image and the configuration hash, and creates the ConfigMap. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-door + namespace: fixture +spec: + replicas: 1 + selector: + matchLabels: + app: nginx-door + template: + metadata: + labels: + app: nginx-door + annotations: + krm-foyer.test/config-hash: CONFIG_HASH + spec: + automountServiceAccountToken: false + # The recipe's backends are this pod's second server (nginx-door.conf). + hostAliases: + - ip: 127.0.0.1 + hostnames: [voter-backend, voter-web] + securityContext: + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx + image: NGINX_IMAGE + command: [nginx, -c, /etc/nginx-door/nginx.conf, -g, "daemon off;"] + ports: + - name: https + containerPort: 8443 + readinessProbe: + tcpSocket: + port: https + periodSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: config + mountPath: /etc/nginx-door + readOnly: true + # The paths the recipe names. + - name: config + mountPath: /etc/nginx/recipe.conf + subPath: recipe.conf + readOnly: true + - name: config + mountPath: /etc/nginx/foyer.js + subPath: foyer.js + readOnly: true + - name: config + mountPath: /etc/nginx/krm-foyer-ca.crt + subPath: ca.crt + readOnly: true + - name: tls + mountPath: /etc/nginx-door/tls + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: nginx-door + - name: tls + secret: + secretName: front-door-tls + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-door + namespace: fixture +spec: + type: NodePort + selector: + app: nginx-door + ports: + - name: https + port: 443 + targetPort: https + nodePort: 30445 diff --git a/test/e2e/cluster/room-pass.sh b/test/e2e/cluster/room-pass.sh new file mode 100755 index 0000000..02b01b4 --- /dev/null +++ b/test/e2e/cluster/room-pass.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Room Pass's QR login in the e2e fixture (docs/room-pass.md): Room Pass 2.0.0 and the +# Dex behind it (room-pass/room-pass.yaml), its CRDs and a Room, a test application with +# the QR entry point (room-pass/room-app.yaml), the two hosts' routes in Traefik +# (room-pass/routes.yaml), and a third krm-foyer on the application's host +# (foyer-room-values.yaml). Run after deploy-foyer.sh and front-door.sh; safe to rerun. +# +# Everything Room Pass's is here and in room-pass/, in the fixture: krm-foyer knows +# nothing of it beyond its generic login settings. +# +# Writes E2E_DIR/room-env: where the suite finds it. +set -euo pipefail + +# Room Pass 2.0.0 (2185782f6f349731909eafdb290007bc3feb168c), the release +# room-pass/crds.yaml is from. +ROOM_PASS_IMAGE="ghcr.io/sunib/room-pass:2.0.0@sha256:eb5fee5161e52eb6de3983c811ddc883838cbee1dfea33fb013374696e189416" +# The same Dex and nginx as start-cluster.sh. +DEX_IMAGE="ghcr.io/dexidp/dex:v2.45.1@sha256:8499afd690c437f52301efd2b05b2455da5bd2dfc20332cd697dc9937f808462" +NGINX_IMAGE="nginx:1.31.6-alpine@sha256:df221db836e1754089190208cee7eeda94f233197056426eda74a43ab1abeac2" +# The application's host, which krm-foyer, Room Pass's /join and the test application +# share, and the issuer's: with the front door's port, which is part of each origin. +ROOM_HOST="room.localhost:8443" +ROOM_PASS_HOST="room-pass.localhost:8443" +# The issuer's address for the API server and pods; start-cluster.sh maps the name here. +ROOM_PASS_SERVICE_IP="10.43.0.212" + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(cd "$here/../../.." && pwd)" +E2E_DIR="${E2E_DIR:-$repo/.e2e}" +[ -f "$E2E_DIR/foyer-env" ] || { echo "krm-foyer is not deployed; run task e2e-deploy first" >&2; exit 1; } +# shellcheck source=/dev/null +. "$E2E_DIR/foyer-env" +export KUBECONFIG="$E2E_DIR/kubeconfig" + +echo "== certificates" +mkdir -p "$E2E_DIR/room" +# cert NAME HOST...: a certificate for every HOST at room/NAME.crt, from the fixture CA, +# renewed when it expires within a day, or the CA or the names changed. +cert() { + local name="$1" crt="$E2E_DIR/room/$1.crt" key="$E2E_DIR/room/$1.key" fresh=1 host sans="" + shift + [ -f "$crt" ] && openssl x509 -checkend 86400 -noout -in "$crt" >/dev/null 2>&1 \ + && openssl verify -CAfile "$E2E_DIR/ca.crt" "$crt" >/dev/null 2>&1 || fresh="" + for host in "$@"; do + sans="$sans${sans:+,}DNS:$host" + [ -n "$fresh" ] && openssl x509 -noout -checkhost "$host" -in "$crt" | grep -q 'does match' || fresh="" + done + [ -n "$fresh" ] && return 0 + openssl req -newkey rsa:2048 -nodes -subj "/CN=$1" -keyout "$key" -out "$E2E_DIR/room/$name.csr" 2>/dev/null + openssl x509 -req -in "$E2E_DIR/room/$name.csr" -CA "$E2E_DIR/ca.crt" -CAkey "$E2E_DIR/ca.key" \ + -CAcreateserial -days 30 -extfile <(printf 'subjectAltName=%s' "$sans") -out "$crt" 2>/dev/null +} +cert edge room.localhost room-pass.localhost +cert krm-foyer-room room.localhost krm-foyer-room.krm-foyer.svc +cert room-app room-app.fixture.svc +tls() { kubectl -n "$1" create secret tls "$2" --cert "$E2E_DIR/room/$3.crt" --key "$E2E_DIR/room/$3.key" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null; } + +echo "== Room Pass's CRDs" +kubectl apply -f "$here/room-pass/crds.yaml" >/dev/null +kubectl wait --for=condition=Established crd/rooms.room-pass.koudijs.dev crd/participants.room-pass.koudijs.dev \ + --timeout=60s >/dev/null + +echo "== Room Pass and its Dex" +fill() { + sed -e "s|ROOM_PASS_IMAGE|$ROOM_PASS_IMAGE|" -e "s|DEX_IMAGE|$DEX_IMAGE|" -e "s|NGINX_IMAGE|$NGINX_IMAGE|" \ + -e "s|ROOM_PASS_HOST|$ROOM_PASS_HOST|g" -e "s|ROOM_HOST|$ROOM_HOST|g" \ + -e "s|ROOM_PASS_SERVICE_IP|$ROOM_PASS_SERVICE_IP|" -e "s|CONFIG_HASH|${config_hash:-}|" "$@" +} +kubectl create namespace room-pass --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# The keys Room Pass seals its cookies with, made once and kept, so its participants stay +# enrolled across reruns. +if ! kubectl -n room-pass get secret room-pass-cookie >/dev/null 2>&1; then + kubectl -n room-pass create secret generic room-pass-cookie \ + --from-file=hash-key=<(head -c 32 /dev/urandom) --from-file=block-key=<(head -c 32 /dev/urandom) >/dev/null +fi +fill "$here/room-pass/dex-config.yaml" > "$E2E_DIR/room/dex-config.yaml" +kubectl -n room-pass create configmap dex --from-file=config.yaml="$E2E_DIR/room/dex-config.yaml" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +# A month from now, every run: a Room that has ended takes no one. +fill "$here/room-pass/room.yaml" | sed "s|ENDS_AT|$(date -u -d '+30 days' +%Y-%m-%dT%H:%M:%SZ)|" \ + | kubectl apply -f - >/dev/null +config_hash="$(cat "$E2E_DIR/room/dex-config.yaml" "$here/room-pass/room-pass.yaml" | sha256sum | cut -c1-16)" +fill "$here/room-pass/room-pass.yaml" | kubectl apply -f - >/dev/null +for d in dex room-pass; do + if ! kubectl -n room-pass rollout status "deployment/$d" --timeout=180s; then + kubectl -n room-pass describe pods -l "app=$d" >&2 + kubectl -n room-pass logs "deployment/$d" --tail=50 >&2 || true + exit 1 + fi +done + +echo "== the test application" +kubectl -n fixture create configmap room-app --from-file=nginx.conf="$here/room-pass/room-app-nginx.conf" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +tls fixture room-app-tls room-app +config_hash="$(cat "$here/room-pass/room-app-nginx.conf" "$E2E_DIR/room/room-app.crt" | sha256sum | cut -c1-16)" +fill "$here/room-pass/room-app.yaml" | kubectl apply -f - >/dev/null +if ! kubectl -n fixture rollout status deployment/room-app --timeout=120s; then + kubectl -n fixture describe pods -l app=room-app >&2 + kubectl -n fixture logs deployment/room-app --tail=50 >&2 || true + exit 1 +fi + +echo "== routes for $ROOM_HOST and $ROOM_PASS_HOST" +for ns in room-pass krm-foyer fixture; do tls "$ns" room-edge-tls edge; done +fill "$here/room-pass/routes.yaml" | kubectl apply -f - >/dev/null + +echo "== krm-foyer on $ROOM_HOST" +apply() { kubectl -n krm-foyer "$@" --dry-run=client -o yaml | kubectl apply -f - >/dev/null; } +tls krm-foyer krm-foyer-room-tls krm-foyer-room +# The client secret of krm-foyer in room-pass/dex-config.yaml. +apply create secret generic krm-foyer-room-oidc --from-literal=client-secret=krm-foyer-room-e2e-secret +if ! kubectl -n krm-foyer get secret krm-foyer-room-session-keys >/dev/null 2>&1; then + kubectl -n krm-foyer create secret generic krm-foyer-room-session-keys \ + --from-literal=session-keys="$(head -c 32 /dev/urandom | base64)" >/dev/null +fi +config_hash="$(cat "$E2E_DIR/room/krm-foyer-room.crt" "$E2E_DIR/ca.crt" | sha256sum | cut -c1-16)" +helm upgrade --install krm-foyer-room "$repo/charts/krm-foyer" --namespace krm-foyer \ + -f "$here/foyer-room-values.yaml" \ + --set-string image.repository="${FOYER_IMAGE%:*}" --set-string image.tag="${FOYER_IMAGE#*:}" \ + --set-string "podAnnotations.krm-foyer\.test/config-hash=$config_hash" >/dev/null +# Ready once it has read the issuer's discovery document, through Traefik and Room Pass. +if ! kubectl -n krm-foyer rollout status deployment/krm-foyer-room --timeout=180s; then + kubectl -n krm-foyer describe pods -l app.kubernetes.io/instance=krm-foyer-room >&2 + kubectl -n krm-foyer logs deployment/krm-foyer-room --tail=50 >&2 || true + exit 1 +fi + +echo "== checks, through the front door" +# As a browser reaches both hosts: the issuer's discovery, krm-foyer's 401 on the +# application's host, the join endpoint refusing a code Room Pass would ignore, and Dex +# unreachable at a path Room Pass does not forward. Traefik takes a moment to load new +# routes, so each is given 30 seconds. +front() { curl -sS --cacert "$E2E_DIR/ca.crt" --resolve "room.localhost:8443:127.0.0.1" \ + --resolve "room-pass.localhost:8443:127.0.0.1" -o /dev/null -w '%{http_code}' "$@"; } +expect_code() { + local url="$1" want="$2" code="" + for _ in $(seq 1 30); do + code="$(front "$url" || true)" + [ "$code" = "$want" ] && return 0 + sleep 1 + done + echo "$url answered $code, want $want" >&2 + exit 1 +} +expect_code "https://$ROOM_PASS_HOST/.well-known/openid-configuration" 200 +expect_code "https://$ROOM_PASS_HOST/not-a-dex-path" 404 +expect_code "https://$ROOM_HOST/auth/session" 401 +expect_code "https://$ROOM_HOST/join-room?code=not-a-code" 400 +expect_code "https://$ROOM_HOST/room/" 200 + +cat > "$E2E_DIR/room-env" <" diff --git a/test/e2e/cluster/room-pass/crds.yaml b/test/e2e/cluster/room-pass/crds.yaml new file mode 100644 index 0000000..2c4eaf6 --- /dev/null +++ b/test/e2e/cluster/room-pass/crds.yaml @@ -0,0 +1,390 @@ +# Room Pass's CRDs, as released in v2.0.0 (2185782f6f349731909eafdb290007bc3feb168c, +# config/crd/), the release whose image room-pass.yaml pins. Copied unchanged; replace +# both together. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: rooms.room-pass.koudijs.dev +spec: + group: room-pass.koudijs.dev + names: + kind: Room + listKind: RoomList + plural: rooms + singular: room + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.joinCode.code + name: Code + type: string + - jsonPath: .spec.enrollment + name: Enrollment + type: string + - jsonPath: .status.participantCount + name: Participants + type: integer + - jsonPath: .spec.endsAt + name: Ends + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + allowedReturnURLs: + description: AllowedReturnURLs must also appear in the deployment + allowlist. + items: + maxLength: 2048 + pattern: ^https://[^/?#@]+/[^#]*$ + type: string + maxItems: 16 + minItems: 1 + type: array + x-kubernetes-list-type: set + x-kubernetes-validations: + - message: allowedReturnURLs is immutable + rule: self == oldSelf + attributionNote: + description: |- + AttributionNote is the operator's one sentence about what the issued + address is for. Room Pass knows the address is synthetic and never a real + mailbox; what the application behind it does with the address -- put it on + a Git commit, write it to an audit log, nothing at all -- is not Room + Pass's to claim. Empty means the join page claims nothing beyond the + address being unroutable. + + It is appended to a sentence that already ends "...it is never a real + mailbox", so say only what Room Pass cannot: "It labels your changes in + Git." Repeating the mailbox clause reads as a stutter on the page. + maxLength: 200 + pattern: ^[^<>\x00-\x1f\x7f]*$ + type: string + audienceGroup: + description: AudienceGroup selects operator-managed RBAC; Room Pass + never creates grants. + maxLength: 128 + pattern: ^demo:[a-zA-Z0-9][a-zA-Z0-9:_-]*$ + type: string + x-kubernetes-validations: + - message: audienceGroup is immutable + rule: self == oldSelf + endsAt: + description: EndsAt can be extended without changing participant identities. + format: date-time + type: string + enrollment: + default: Closed + enum: + - Open + - Closed + type: string + joinCode: + default: + length: 6 + rotateEvery: 15s + validFor: 30s + description: JoinCodeSpec controls rotation and the grace period for + typing a previous code. + properties: + length: + default: 6 + maximum: 12 + minimum: 6 + type: integer + rotateEvery: + default: 15s + enum: + - 10s + - 15s + - 30s + - 60s + type: string + validFor: + default: 30s + enum: + - 20s + - 30s + - 45s + - 60s + - 90s + - 120s + - 180s + - 240s + type: string + type: object + x-kubernetes-validations: + - message: joinCode is immutable + rule: self == oldSelf + - message: validFor must provide at least five seconds overlap and + retain at most four codes + rule: duration(self.validFor) >= duration(self.rotateEvery) + duration('5s') + && duration(self.validFor) <= duration(self.rotateEvery) + duration(self.rotateEvery) + + duration(self.rotateEvery) + duration(self.rotateEvery) + maxParticipants: + maximum: 10000 + minimum: 1 + type: integer + stopped: + default: false + description: Stopped permanently denies enrollment and new identity + assertions. + type: boolean + x-kubernetes-validations: + - message: stopping is irreversible + rule: '!oldSelf || self' + title: + description: Title appears on the participant join page. + maxLength: 120 + minLength: 1 + type: string + required: + - allowedReturnURLs + - audienceGroup + - endsAt + - enrollment + - joinCode + - maxParticipants + - stopped + - title + type: object + status: + properties: + conditions: + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + joinCode: + properties: + code: + maxLength: 12 + type: string + expiresAt: + format: date-time + type: string + issuedAt: + format: date-time + type: string + required: + - code + - expiresAt + - issuedAt + type: object + observedGeneration: + format: int64 + type: integer + participantCount: + type: integer + validJoinCodes: + description: ValidJoinCodes is operator-only credential material, + never a public response. + items: + properties: + code: + maxLength: 12 + type: string + expiresAt: + format: date-time + type: string + issuedAt: + format: date-time + type: string + required: + - code + - expiresAt + - issuedAt + type: object + maxItems: 4 + type: array + required: + - participantCount + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.22.0 + name: participants.room-pass.koudijs.dev +spec: + group: room-pass.koudijs.dev + names: + kind: Participant + listKind: ParticipantList + plural: participants + singular: participant + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.roomRef.name + name: Room + type: string + - jsonPath: .spec.revoked + name: Revoked + type: boolean + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + properties: + displayName: + description: |- + DisplayName is an unverified demo label, never an authorization key. + + It is stored in its FOLDED form (see labelName in internal/server): what a + participant types is normalized before it reaches here, so this value is + legal in all three places it travels to -- a Kubernetes label value on + every ballot the voter app writes, one path segment in the mirrored audit + trail, and the tail of a submission's object name. The pattern is exactly + what the fold emits and is deliberately narrower than a label value: no + "_" or ".", because an object name is DNS-1123 and rejects the first. + + MaxLength matches maxParticipantID rather than a label value's 63, so the + display name and the address derived from it share one limit instead of + two that disagree past forty characters. + maxLength: 40 + minLength: 1 + pattern: ^[A-Za-z0-9]([-A-Za-z0-9]*[A-Za-z0-9])?$ + type: string + x-kubernetes-validations: + - message: displayName is immutable + rule: self == oldSelf + revoked: + default: false + type: boolean + x-kubernetes-validations: + - message: revocation is irreversible + rule: '!oldSelf || self' + roomRef: + properties: + name: + maxLength: 253 + minLength: 1 + type: string + uid: + description: UID prevents name reuse from reviving enrollment. + maxLength: 128 + minLength: 1 + type: string + required: + - name + - uid + type: object + x-kubernetes-validations: + - message: roomRef is immutable + rule: self == oldSelf + required: + - displayName + - revoked + - roomRef + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/test/e2e/cluster/room-pass/dex-config.yaml b/test/e2e/cluster/room-pass/dex-config.yaml new file mode 100644 index 0000000..a6d8ad1 --- /dev/null +++ b/test/e2e/cluster/room-pass/dex-config.yaml @@ -0,0 +1,36 @@ +# Room Pass's Dex (deploy/example/dex-config.yaml), with krm-foyer as its client. +# room-pass.sh fills in the hosts. +issuer: https://ROOM_PASS_HOST +storage: + type: sqlite3 + config: + file: /var/dex/dex.db +# No rotation while a fixture lives: a new key means minutes of refused tokens until the +# API server fetches it. +expiry: + signingKeys: 8760h +web: + http: 0.0.0.0:5556 +oauth2: + skipApprovalScreen: true +staticClients: + # krm-foyer-room (foyer-room-values.yaml) on the application's host. It asks for the + # groups and federated:id scopes: the API server refuses a token without + # federated_claims. + - id: krm-foyer + name: krm-foyer + secret: krm-foyer-room-e2e-secret + redirectURIs: + - https://ROOM_HOST/auth/callback +connectors: + # The only connector Room Pass works with; its id sets the callback path Room Pass + # serves, /callback/room-pass. + - type: authproxy + id: room-pass + name: Join the room + config: + userIDHeader: X-Remote-User-Id + userHeader: X-Remote-User + userNameHeader: X-Remote-User-Name + emailHeader: X-Remote-User-Email + groupHeader: X-Remote-Group diff --git a/test/e2e/cluster/room-pass/room-app-nginx.conf b/test/e2e/cluster/room-pass/room-app-nginx.conf new file mode 100644 index 0000000..ccb0e96 --- /dev/null +++ b/test/e2e/cluster/room-pass/room-app-nginx.conf @@ -0,0 +1,63 @@ +# The test application on room.localhost (routes.yaml): the application's side of Room +# Pass's QR join, and a page to come back to. Nothing in it is krm-foyer's. +worker_processes 1; +error_log /dev/stderr warn; +pid /tmp/nginx.pid; + +events {} + +http { + access_log /dev/stdout; + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server_tokens off; + absolute_redirect off; + + server { + listen 8443 ssl; + ssl_certificate /etc/room-app/tls/tls.crt; + ssl_certificate_key /etc/room-app/tls/tls.key; + ssl_protocols TLSv1.3; + + # A tripwire, as in hello-web: no cookie of krm-foyer's or Room Pass's gets here. + if ($http_cookie) { + return 400; + } + + add_header Content-Security-Policy "default-src 'self'; object-src 'none'; base-uri 'none'; form-action 'self'; frame-ancestors 'none'" always; + add_header X-Content-Type-Options nosniff always; + add_header Cache-Control no-store always; + + # GET /join-room?code=K7Q2: what the QR code on the presenter's screen points at. + # Room Pass's cookie contract (its docs/qr-join.md): the code, bounded to what Room + # Pass accepts, in __Host-room-pass-joincode, Secure, HttpOnly, Path=/ and + # SameSite=Lax (Strict would not survive the redirects through the issuer's host), + # for five minutes. Then the application's own login: krm-foyer's, with Room Pass's + # connector and a destination fixed here, never taken from the request. The code + # goes nowhere else: not into the login link, and not into a log, since it is a + # credential while it is valid. + location = /join-room { + access_log off; + add_header Referrer-Policy no-referrer always; + add_header Cache-Control no-store always; + if ($arg_code !~ "^[A-Z0-9]{1,12}$") { + return 400 "not a room code\n"; + } + add_header Set-Cookie "__Host-room-pass-joincode=$arg_code; Path=/; Max-Age=300; Secure; HttpOnly; SameSite=Lax" always; + return 302 "/auth/login?return_to=%2Froom%2F&oidc.connector_id=room-pass"; + } + + # Where a participant arrives, signed in. + location = /room/ { + default_type text/html; + return 200 'The room

You are in the room.

'; + } + + location / { + return 404; + } + } +} diff --git a/test/e2e/cluster/room-pass/room-app.yaml b/test/e2e/cluster/room-pass/room-app.yaml new file mode 100644 index 0000000..d22e01d --- /dev/null +++ b/test/e2e/cluster/room-pass/room-app.yaml @@ -0,0 +1,72 @@ +# The test application's nginx (room-app-nginx.conf). room-pass.sh fills in the image and +# the configuration hash, and creates the ConfigMap and the Secret. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: room-app + namespace: fixture +spec: + replicas: 1 + selector: + matchLabels: + app: room-app + template: + metadata: + labels: + app: room-app + annotations: + krm-foyer.test/config-hash: CONFIG_HASH + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx + image: NGINX_IMAGE + command: [nginx, -c, /etc/room-app/config/nginx.conf, -g, "daemon off;"] + ports: + - name: https + containerPort: 8443 + readinessProbe: + tcpSocket: + port: https + periodSeconds: 2 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + volumeMounts: + - name: config + mountPath: /etc/room-app/config + readOnly: true + - name: tls + mountPath: /etc/room-app/tls + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: config + configMap: + name: room-app + - name: tls + secret: + secretName: room-app-tls + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: room-app + namespace: fixture +spec: + selector: + app: room-app + ports: + - name: https + port: 443 + targetPort: https diff --git a/test/e2e/cluster/room-pass/room-pass.yaml b/test/e2e/cluster/room-pass/room-pass.yaml new file mode 100644 index 0000000..952cbbb --- /dev/null +++ b/test/e2e/cluster/room-pass/room-pass.yaml @@ -0,0 +1,249 @@ +# Room Pass 2.0.0 and the Dex behind it, as Room Pass's own reference deployment +# (deploy/base, deploy/dex) has them, with the fixture's names. room-pass.sh fills in +# the images, the hosts and the configuration hash, and creates the ConfigMap `dex` and +# the Secret `room-pass-cookie`. +# +# ROOM_HOST the application's host, which Room Pass shares: /join, /bind, /logout +# ROOM_PASS_HOST the issuer's host: all Room Pass's, which forwards only Dex's +# protocol paths to Dex +apiVersion: v1 +kind: Namespace +metadata: + name: room-pass +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: room-pass + namespace: room-pass +--- +# Room Pass's Role, from deploy/base/rbac.yaml: its Room, the Participants it enrolls, +# and its cookie keys by name. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: room-pass + namespace: room-pass +rules: + - apiGroups: [room-pass.koudijs.dev] + resources: [rooms] + verbs: [get, list, watch] + - apiGroups: [room-pass.koudijs.dev] + resources: [rooms/status] + verbs: [get, update, patch] + - apiGroups: [room-pass.koudijs.dev] + resources: [participants] + verbs: [get, list, create] + - apiGroups: [""] + resources: [secrets] + resourceNames: [room-pass-cookie] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: room-pass + namespace: room-pass +subjects: + - kind: ServiceAccount + name: room-pass + namespace: room-pass +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: room-pass +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: room-pass + namespace: room-pass +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: room-pass + template: + metadata: + labels: + app: room-pass + annotations: + krm-foyer.test/config-hash: CONFIG_HASH + spec: + serviceAccountName: room-pass + securityContext: + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: room-pass + image: ROOM_PASS_IMAGE + ports: + - name: http + containerPort: 8080 + - name: metrics + containerPort: 9090 + env: + # The Room this instance serves, in its own namespace (room.yaml). + - name: ROOM_NAME + value: e2e + - name: ROOM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: DEX_UPSTREAM + value: http://dex:5556 + # Origins, port included: Room Pass routes by Host, and the browser's Host + # carries the port of the front door. + - name: JOIN_ORIGIN + value: https://ROOM_HOST + - name: ISSUER_ORIGIN + value: https://ROOM_PASS_HOST + - name: ALLOWED_RETURN_URLS + value: https://ROOM_HOST/ + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /readyz + port: http + periodSeconds: 2 +--- +apiVersion: v1 +kind: Service +metadata: + name: room-pass + namespace: room-pass +spec: + selector: + app: room-pass + ports: + - name: http + port: 8080 + targetPort: http +--- +# Dex as Room Pass's issuer, from deploy/dex: one replica on SQLite, so its signing keys +# outlive a restart of the pod. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: dex + namespace: room-pass +spec: + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 64Mi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dex + namespace: room-pass +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: dex + template: + metadata: + labels: + app: dex + annotations: + krm-foyer.test/config-hash: CONFIG_HASH + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + containers: + - name: dex + image: DEX_IMAGE + args: [dex, serve, /etc/dex/config.yaml] + ports: + - name: http + containerPort: 5556 + volumeMounts: + - name: config + mountPath: /etc/dex + readOnly: true + - name: data + mountPath: /var/dex + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + readinessProbe: + httpGet: + path: /.well-known/openid-configuration + port: http + periodSeconds: 2 + volumes: + - name: config + configMap: + name: dex + - name: data + persistentVolumeClaim: + claimName: dex + - name: tmp + emptyDir: + sizeLimit: 16Mi +--- +# Cluster-internal only: nothing routes to it, and Room Pass forwards only Dex's protocol +# paths to it from the issuer's host. +apiVersion: v1 +kind: Service +metadata: + name: dex + namespace: room-pass +spec: + selector: + app: dex + ports: + - name: http + port: 5556 + targetPort: http +--- +# The boundary that makes the authproxy connector safe (deploy/dex/networkpolicy.yaml): +# authproxy believes the X-Remote-* headers on its callback, so only Room Pass, which +# sets them after enrolling a participant, may reach Dex at all. k3s enforces +# NetworkPolicy; the suite checks that another pod cannot connect, and that a pod +# labelled as Room Pass can. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: dex-only-from-room-pass + namespace: room-pass +spec: + podSelector: + matchLabels: + app: dex + policyTypes: [Ingress] + ingress: + - from: + - podSelector: + matchLabels: + app: room-pass + ports: + - protocol: TCP + port: 5556 diff --git a/test/e2e/cluster/room-pass/room.yaml b/test/e2e/cluster/room-pass/room.yaml new file mode 100644 index 0000000..1e946eb --- /dev/null +++ b/test/e2e/cluster/room-pass/room.yaml @@ -0,0 +1,22 @@ +# The event: one Room, which Room Pass serves (ROOM_NAME e2e). room-pass.sh fills in +# the host and when it ends, a month from each run. Its join code rotates every 30 +# seconds and stays valid for 120, as Room Pass's docs/qr-join.md advises for a QR flow. +apiVersion: room-pass.koudijs.dev/v1alpha1 +kind: Room +metadata: + name: e2e + namespace: room-pass +spec: + title: krm-foyer e2e + attributionNote: Your name labels the changes you make in the e2e suite. + endsAt: "ENDS_AT" + enrollment: Open + maxParticipants: 1000 + # Every participant's group; the suite binds its grants to it. + audienceGroup: demo:krm-foyer-e2e + allowedReturnURLs: + - https://ROOM_HOST/ + joinCode: + rotateEvery: 30s + validFor: 120s + length: 6 diff --git a/test/e2e/cluster/room-pass/routes.yaml b/test/e2e/cluster/room-pass/routes.yaml new file mode 100644 index 0000000..025a1fb --- /dev/null +++ b/test/e2e/cluster/room-pass/routes.yaml @@ -0,0 +1,141 @@ +# The two hosts of Room Pass's QR login (docs/room-pass.md), in Traefik's IngressRoutes, +# each in the namespace of the Service it routes to: +# +# room.localhost the application's host, shared by four owners: +# /auth/ /k8s/ /stream/ /_foyer/ krm-foyer-room +# /join /bind /logout (exact) Room Pass +# everything else, /join-room too the test application (room-app) +# room-pass.localhost the issuer's host: all of it Room Pass's, which forwards only +# Dex's protocol paths to Dex +# +# A browser reaches both through the front door's port-forward, on 127.0.0.1:8443. The +# API server and krm-foyer reach the issuer's host at the fixed address of the Service +# room-pass-issuer below, in front of Traefik (start-cluster.sh puts its name there). +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: strip-identity-headers + namespace: room-pass +spec: + # Room Pass's deploy/edge/traefik/middlewares.yaml: Dex's authproxy connector turns + # X-Remote-* into an identity, so nothing from a browser may carry them. Room Pass + # removes them too; this is the outer of the two controls. + headers: + customRequestHeaders: + X-Remote-User: "" + X-Remote-User-Id: "" + X-Remote-User-Name: "" + X-Remote-User-Email: "" + X-Remote-Group: "" +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: room-pass + namespace: room-pass +spec: + entryPoints: [websecure] + routes: + - match: Host(`room.localhost`) && (Path(`/join`) || Path(`/bind`) || Path(`/logout`)) + middlewares: + - name: strip-identity-headers + services: + - name: room-pass + port: 8080 + - match: Host(`room-pass.localhost`) + middlewares: + - name: strip-identity-headers + services: + - name: room-pass + port: 8080 + tls: + secretName: room-edge-tls +--- +# Traefik to krm-foyer-room is TLS, verified by its Service name, as for the main one. +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: krm-foyer-room + namespace: krm-foyer +spec: + serverName: krm-foyer-room.krm-foyer.svc + rootCAs: + - configMap: krm-foyer-issuer-ca +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: krm-foyer-room + namespace: krm-foyer +spec: + entryPoints: [websecure] + routes: + - match: Host(`room.localhost`) && (PathPrefix(`/auth/`) || PathPrefix(`/k8s/`) || PathPrefix(`/stream/`) || PathPrefix(`/_foyer/`)) + services: + - name: krm-foyer-room + port: 443 + scheme: https + serversTransport: krm-foyer-room + tls: + secretName: room-edge-tls +--- +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: room-app + namespace: fixture +spec: + serverName: room-app.fixture.svc + rootCAs: + - configMap: hello-web-ca +--- +# The test application: its pages and its QR entry point. It has no use for cookies; +# Traefik removes them (the no-cookie Middleware of traefik-routes.yaml), and its +# tripwire refuses any that arrive. +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: room-app + namespace: fixture +spec: + entryPoints: [websecure] + routes: + # The QR code's target carries the join code in its query, and the code is a + # credential while it is valid: this route keeps it out of Traefik's access log. + - match: Host(`room.localhost`) && Path(`/join-room`) + observability: + accessLogs: false + middlewares: + - name: no-cookie + services: + - name: room-app + port: 443 + scheme: https + serversTransport: room-app + - match: Host(`room.localhost`) + middlewares: + - name: no-cookie + services: + - name: room-app + port: 443 + scheme: https + serversTransport: room-app + tls: + secretName: room-edge-tls +--- +# The issuer's host for the API server and for pods, which cannot use the browser's +# port-forward: Traefik's websecure port at a fixed address, which start-cluster.sh maps +# room-pass.localhost to. Port 8443, so the issuer URL is the same everywhere. +apiVersion: v1 +kind: Service +metadata: + name: room-pass-issuer + namespace: traefik-system +spec: + clusterIP: ROOM_PASS_SERVICE_IP + selector: + app.kubernetes.io/name: traefik + ports: + - name: websecure + port: 8443 + targetPort: 8443 diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index ef4fb93..36cf353 100755 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -31,12 +31,18 @@ GATEWAY="${GATEWAY:-172.29.250.1}" # API server all use one issuer URL. DEX_HOST="dex.localhost" ISSUER_HOST="issuer.krm-foyer.test" +# Room Pass's issuer (room-pass.sh): Traefik's address, for the room-pass.localhost route. +ROOM_PASS_HOST="room-pass.localhost" ISSUER_URL="https://$ISSUER_HOST:8443" # Fixed ClusterIPs in k3s's default Service range (10.43.0.0/16). The API server runs on -# the node, not in a pod, so it cannot use cluster DNS; --host-alias puts these in the -# node's /etc/hosts (and in CoreDNS, for pods) when the cluster is created. +# the node, not in a pod, so it cannot use cluster DNS; host_aliases below puts these in +# the node's /etc/hosts, and in CoreDNS for pods. DEX_SERVICE_IP="10.43.200.10" ISSUER_SERVICE_IP="10.43.200.11" +# In the band at the start of the Service range that Kubernetes keeps for fixed +# addresses: this Service is created late (room-pass.sh), after the cluster has handed +# out addresses of its own, and one could already be 10.43.200.12. +ROOM_PASS_SERVICE_IP="10.43.0.212" VOLUME="${CLUSTER_NAME}-config" SERVER_CONTAINER="k3d-${CLUSTER_NAME}-server-0" # The same k3s release gitops-reverser's e2e runs on. @@ -122,8 +128,6 @@ if ! k3d cluster get "$CLUSTER_NAME" >/dev/null 2>&1; then k3d cluster create "$CLUSTER_NAME" \ --image "$K3S_IMAGE" --servers 1 --agents 0 --wait --timeout 180s \ --network "$NETWORK" --no-lb --api-port 127.0.0.1:0 \ - --host-alias "$DEX_SERVICE_IP:$DEX_HOST" \ - --host-alias "$ISSUER_SERVICE_IP:$ISSUER_HOST" \ --kubeconfig-update-default=false --kubeconfig-switch-context=false \ --volume "$VOLUME:/etc/krm-foyer-e2e@server:0" \ --k3s-arg "--disable=traefik,servicelb,metrics-server@server:0" \ @@ -131,10 +135,6 @@ if ! k3d cluster get "$CLUSTER_NAME" >/dev/null 2>&1; then --k3s-arg "--kube-apiserver-arg=audit-policy-file=/etc/krm-foyer-e2e/audit-policy.yaml@server:0" \ --k3s-arg "--kube-apiserver-arg=audit-log-path=/etc/krm-foyer-e2e/audit.log@server:0" \ --k3s-arg "--kube-apiserver-arg=audit-log-maxsize=50@server:0" -elif ! docker exec "$SERVER_CONTAINER" grep -q "^${DEX_SERVICE_IP}[[:space:]].*${DEX_HOST}" /etc/hosts; then - # --host-alias is fixed at creation, so a cluster from an older script cannot find Dex. - echo "cluster $CLUSTER_NAME has no alias $DEX_HOST -> $DEX_SERVICE_IP (made by an older script); run task e2e-down" >&2 - exit 1 elif [ "$(cat "$E2E_DIR/apiserver.hash" 2>/dev/null)" != "$apiserver_hash" ]; then echo "the API server's configuration changed; restarting it" put_config @@ -171,6 +171,38 @@ for _ in $(seq 1 45); do done kubectl wait --for=condition=Ready "node/$SERVER_CONTAINER" --timeout=90s >/dev/null +echo "== host aliases" +# The issuers' names, at their Services' fixed addresses: in the node's /etc/hosts for +# the API server, and in CoreDNS for pods. Docker rewrites /etc/hosts when the node +# restarts, which a changed authentication configuration causes, so the entries are put +# back on every run. For pods they are a server block of CoreDNS's own, in the +# coredns-custom ConfigMap that k3s's Corefile imports and k3s leaves alone; NodeHosts, +# where k3d's --host-alias writes, is rewritten by k3s at every start. +host_aliases() { + local entry ip name zones="" hosts="" applied + for entry in "$@"; do + ip="${entry%%:*}" name="${entry#*:}" + docker exec "$SERVER_CONTAINER" grep -q "^${ip}[[:space:]]\+${name}\$" /etc/hosts \ + || docker exec "$SERVER_CONTAINER" sh -c "echo '$ip $name' >> /etc/hosts" + zones="$zones${zones:+ }$name:53" + hosts="$hosts $ip $name"$'\n' + done + applied="$(kubectl -n kube-system create configmap coredns-custom \ + --from-literal=fixture-issuers.server="$zones {"$'\n'" hosts {"$'\n'"$hosts }"$'\n'"}"$'\n' \ + --dry-run=client -o yaml | kubectl apply -f -)" + case "$applied" in + *unchanged*) ;; + *) + # A new pod reads the ConfigMap as it is now. On a new cluster k3s creates CoreDNS a + # moment after the API server answers. + for _ in $(seq 1 45); do kubectl -n kube-system get deployment coredns >/dev/null 2>&1 && break; sleep 2; done + kubectl -n kube-system rollout restart deployment/coredns >/dev/null + kubectl -n kube-system rollout status deployment/coredns --timeout=120s >/dev/null + ;; + esac +} +host_aliases "$DEX_SERVICE_IP:$DEX_HOST" "$ISSUER_SERVICE_IP:$ISSUER_HOST" "$ROOM_PASS_SERVICE_IP:$ROOM_PASS_HOST" + echo "== issuers: Dex at https://$DEX_HOST:5556, test issuer at $ISSUER_URL" # The test issuer serves its discovery document and the public half of the signing key. modulus="$(openssl rsa -in "$E2E_DIR/issuer-signing.key" -noout -modulus 2>/dev/null | cut -d= -f2 \ diff --git a/test/e2e/cluster/traefik-routes.yaml b/test/e2e/cluster/traefik-routes.yaml new file mode 100644 index 0000000..2bdab4f --- /dev/null +++ b/test/e2e/cluster/traefik-routes.yaml @@ -0,0 +1,104 @@ +# The Traefik recipe of docs/ingress.md, made real: the same origin as gateway.yaml, +# with two routes Gateway API cannot express without an experimental filter, written as +# Traefik's own IngressRoute and Middlewares, as a cluster that routes with Traefik +# (Voter's does) would write them. +# +# /public/ a domain backend, behind /auth/check?identity=true: Traefik asks krm-foyer +# who the user is, and passes the answer on in Krm-Foyer-Identity +# /members/ pages behind the login gate, /auth/check?redirect=true: a signed-out page +# load goes to the login, and comes back +# +# Both are served by hello-web's nginx (hello-web-nginx.conf), which stands in for a +# backend: /public/whoami echoes the identity header it received. front-door.sh applies +# this file after gateway.yaml. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: foyer-identity + namespace: fixture +spec: + forwardAuth: + # krm-foyer's Service, by a name its certificate holds (deploy-foyer.sh), verified + # against the fixture CA. The check gets the browser's own request headers (Cookie, + # Origin, the CSRF proof) and Traefik's X-Forwarded-Method and X-Forwarded-Uri. + address: https://krm-foyer.krm-foyer.svc/auth/check?identity=true + tls: + caSecret: foyer-check-ca + # The check's answer is a Status or krm-foyer's sign-in page: small. + maxResponseBodySize: 65536 + # Traefik removes the browser's copy of each header named here before it copies + # krm-foyer's, so a forged Krm-Foyer-Identity never reaches the backend. The suite + # sends one to make sure. + authResponseHeaders: [Krm-Foyer-Identity] + # X-Forwarded-* from the browser are not to be believed: Traefik sets its own. + trustForwardHeader: false +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: foyer-gate + namespace: fixture +spec: + forwardAuth: + address: https://krm-foyer.krm-foyer.svc/auth/check?redirect=true + tls: + caSecret: foyer-check-ca + maxResponseBodySize: 65536 + trustForwardHeader: false +--- +# The backend has no use for the session cookie, so it never leaves Traefik on these +# routes. An empty value removes the header. hello-web's tripwire refuses any request +# with a cookie, so every spec through these routes checks this too. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: no-cookie + namespace: fixture +spec: + headers: + customRequestHeaders: + Cookie: "" +--- +# Traefik to hello-web is TLS, verified, as on the Gateway's route (BackendTLSPolicy). +apiVersion: traefik.io/v1alpha1 +kind: ServersTransport +metadata: + name: hello-web + namespace: fixture +spec: + serverName: hello-web.fixture.svc + rootCAs: + - configMap: hello-web-ca +--- +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: hello-backend + namespace: fixture +spec: + entryPoints: [websecure] + routes: + # The middlewares run in order: the check first, then the cookie goes. An explicit + # priority, so these win over the Gateway's catch-all route to hello-web. + - match: Host(`foyer.localhost`) && PathPrefix(`/public/`) + priority: 1000 + middlewares: + - name: foyer-identity + - name: no-cookie + services: + - name: hello-web + port: 443 + scheme: https + serversTransport: hello-web + - match: Host(`foyer.localhost`) && PathPrefix(`/members/`) + priority: 1000 + middlewares: + - name: foyer-gate + - name: no-cookie + services: + - name: hello-web + port: 443 + scheme: https + serversTransport: hello-web + tls: + secretName: front-door-tls diff --git a/test/e2e/cluster/traefik-values.yaml b/test/e2e/cluster/traefik-values.yaml index 9e0d72e..89ef28b 100644 --- a/test/e2e/cluster/traefik-values.yaml +++ b/test/e2e/cluster/traefik-values.yaml @@ -2,7 +2,8 @@ # here as a Gateway API implementation only. install-traefik.sh installs it with Helm. # # The fixture's routing is Gateway API (gateway.yaml): a Gateway and HTTPRoutes, the -# portable form docs/ingress.md recommends, rather than Traefik's own IngressRoute. +# portable form docs/ingress.md recommends. traefik-routes.yaml adds the routes behind +# /auth/check, in Traefik's own IngressRoute, as its recipe there. global: checkNewVersion: false sendAnonymousUsage: false @@ -21,9 +22,12 @@ deployment: providers: kubernetesGateway: enabled: true - # Gateway API only: no Ingress or IngressRoute is read. + # Traefik's own IngressRoute and Middleware too, for the Traefik recipe of + # docs/ingress.md (traefik-routes.yaml): ForwardAuth to /auth/check. Only from the + # namespace a resource is in. kubernetesCRD: - enabled: false + enabled: true + allowCrossNamespace: false kubernetesIngress: enabled: false diff --git a/test/e2e/fixture_test.go b/test/e2e/fixture_test.go index 16ae312..67795bb 100644 --- a/test/e2e/fixture_test.go +++ b/test/e2e/fixture_test.go @@ -50,14 +50,21 @@ type fixture struct { client *http.Client // foyerURL is krm-foyer's public URL, deployed by deploy-foyer.sh, and - // foyerAccount the service account it runs as: cluster-admin, as bait. - foyerURL, foyerNamespace, foyerAccount string + // foyerAccount the service account it runs as: cluster-admin, as bait. foyerAddr + // is the node's address and NodePort the suite reaches it on. + foyerURL, foyerAddr, foyerNamespace, foyerAccount string // sharedAccount is the identity shared watches are opened with: narrow, and not // the bait. sharedAccount string // briefTransport reaches the brief krm-foyer, whose sessions end within a minute, // instead of the main one. briefTransport http.RoundTripper + // frontDoorTransport reaches krm-foyer's name through Traefik, the front door, as + // a browser does: krm-foyer's routes and the application's, on one origin. + frontDoorTransport http.RoundTripper + // nginxTransport reaches krm-foyer's name through docs/ingress.md's nginx recipe + // (nginx-door.conf), in front of the main krm-foyer. + nginxTransport http.RoundTripper // rehearsalUsers is how many rehearsal users Dex has: rehearsal-001@example.com // and on, with alice's password (start-cluster.sh). rehearsalUsers int @@ -81,8 +88,12 @@ func (f *fixture) remember(tokens ...string) { } const ( - password = "password" - foyerClient = "krm-foyer" + password = "password" + foyerClient = "krm-foyer" + // The operator's command line: a client whose tokens the API server maps to + // kubectl:, not oidc: (authentication-config.yaml). + cliClient = "kubectl" + cliSecret = "kubectl-e2e-secret" foyerSecret = "krm-foyer-e2e-secret" otherClient = "other-app" otherSecret = "other-app-e2e-secret" @@ -127,6 +138,7 @@ func loadFixture() *fixture { // Every string contains the empty one: a missing account would pass assertions // that it never appears. Expect(env["FOYER_SHARED_ACCOUNT"]).NotTo(BeEmpty(), "FOYER_SHARED_ACCOUNT in foyer-env (an older deployment? run task e2e-deploy)") + Expect(env["FOYER_NGINX_ADDR"]).NotTo(BeEmpty(), "FOYER_NGINX_ADDR in foyer-env (an older deployment? run task e2e-deploy)") foyer, err := url.Parse(env["FOYER_URL"]) Expect(err).NotTo(HaveOccurred()) @@ -161,11 +173,17 @@ func loadFixture() *fixture { testIssuer: env["TEST_ISSUER"], signingKey: key.(*rsa.PrivateKey), foyerURL: env["FOYER_URL"], + foyerAddr: env["FOYER_ADDR"], foyerNamespace: env["FOYER_NAMESPACE"], foyerAccount: env["FOYER_SERVICE_ACCOUNT"], sharedAccount: env["FOYER_SHARED_ACCOUNT"], briefTransport: transportTo(env["FOYER_BRIEF_ADDR"]), - rehearsalUsers: rehearsalUsers, + // The front door: Traefik, through the port-forward on this container's + // loopback (port-forward.sh), under the same name as krm-foyer, as a browser + // reaches it. + frontDoorTransport: transportTo(net.JoinHostPort("127.0.0.1", foyer.Port())), + nginxTransport: transportTo(env["FOYER_NGINX_ADDR"]), + rehearsalUsers: rehearsalUsers, client: &http.Client{ Transport: transport, Timeout: 30 * time.Second, @@ -190,11 +208,16 @@ func readEnv(file, missing string) map[string]string { // login returns a Dex ID token for user, issued to clientID. It uses the password // grant so the suite can hold a user's own credential and ask the API server directly. func (f *fixture) login(ctx context.Context, user, clientID, secret string) string { + return f.loginScopes(ctx, user, clientID, secret, "openid email profile") +} + +// loginScopes is login, asking Dex for scope. +func (f *fixture) loginScopes(ctx context.Context, user, clientID, secret, scope string) string { form := url.Values{ "grant_type": {"password"}, "username": {user}, "password": {password}, - "scope": {"openid email profile"}, + "scope": {scope}, } req, err := http.NewRequestWithContext(ctx, http.MethodPost, f.dexIssuer+"/token", strings.NewReader(form.Encode())) Expect(err).NotTo(HaveOccurred()) @@ -248,17 +271,24 @@ func b64JSON(v any) string { // selfSubjectReview asks the API server who it takes token for. The answer is 201 with // the username, or the refusal's status code with no username. func (f *fixture) selfSubjectReview(ctx context.Context, token string) (int, string) { + code, name, _ := f.selfSubjectReviewGroups(ctx, token) + return code, name +} + +// selfSubjectReviewGroups is selfSubjectReview with the groups the API server mapped. +func (f *fixture) selfSubjectReviewGroups(ctx context.Context, token string) (int, string, []string) { a := f.direct(ctx, token, http.MethodPost, "/apis/authentication.k8s.io/v1/selfsubjectreviews", []byte(`{"apiVersion":"authentication.k8s.io/v1","kind":"SelfSubjectReview"}`)) var review struct { Status struct { UserInfo struct { - Username string `json:"username"` + Username string `json:"username"` + Groups []string `json:"groups"` } `json:"userInfo"` } `json:"status"` } _ = json.Unmarshal(a.Body, &review) - return a.Code, review.Status.UserInfo.Username + return a.Code, review.Status.UserInfo.Username, review.Status.UserInfo.Groups } // answer is one HTTP response, reduced to what the suite compares. @@ -393,6 +423,13 @@ func (f *fixture) grant(ns, username, resource string, verbs ...string) (revoke return func() { f.kubectl("-n", ns, "delete", "rolebinding", name) } } +// grantGroup is grant, to a Kubernetes group instead of a username. +func (f *fixture) grantGroup(ns, group, resource string, verbs ...string) { + name := "e2e-" + randomID() + f.kubectl("-n", ns, "create", "role", name, "--resource="+resource, "--verb="+strings.Join(verbs, ",")) + f.kubectl("-n", ns, "create", "rolebinding", name, "--role="+name, "--group="+group) +} + // replaceFoyer replaces the main krm-foyer's pod by running kubectl with args in its // namespace (a rollout restart, or a forced delete), and returns once the new pod is // the only one and answers. Sessions are in their cookies, so they survive this. @@ -429,6 +466,44 @@ func (f *fixture) replaceFoyer(ctx context.Context, args ...string) { } return nil }).Should(Succeed()) + f.forgetGonePods() +} + +// forgetGonePods removes the node's connection tracking for krm-foyer's NodePort that +// still leads to a pod that is gone. A connection is tracked with the pod it was sent +// to, and kube-proxy forgets TCP entries only when they time out (up to two minutes). +// Until then a new connection from the same client port can follow the old entry to +// an address nobody answers: a dial timeout. One client making thousands of connections +// a minute, as this suite does, reuses ports that soon after a pod is replaced. So +// whatever replaces a pod forgets it, as kube-proxy does for UDP. +func (f *fixture) forgetGonePods() { + GinkgoHelper() + live := map[string]bool{} + for _, ip := range strings.Fields(f.kubectl("-n", f.foyerNamespace, "get", "pods", + "-o", "jsonpath={.items[*].status.podIP}")) { + live[ip] = true + } + _, port, err := net.SplitHostPort(f.foyerAddr) + Expect(err).NotTo(HaveOccurred()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, "docker", "exec", f.serverContainer, + "conntrack", "-L", "-p", "tcp", "--orig-port-dst", port).Output() + Expect(err).NotTo(HaveOccurred()) + gone := map[string]bool{} + for line := range strings.Lines(string(out)) { + // The reply's source is the pod the connection was sent to. + for _, field := range strings.Fields(line) { + if ip, ok := strings.CutPrefix(field, "src="); ok && strings.HasPrefix(ip, "10.42.") && !live[ip] { + gone[ip] = true + } + } + } + for ip := range gone { + // Exit status 1 when there was nothing left to delete. + _ = exec.CommandContext(ctx, "docker", "exec", f.serverContainer, + "conntrack", "-D", "-p", "tcp", "--orig-port-dst", port, "--reply-src", ip).Run() + } } // kubectl runs as the fixture's admin, for setup only. Nothing a spec asserts on is diff --git a/test/e2e/foyer_auth_test.go b/test/e2e/foyer_auth_test.go index 2424494..74d30a8 100644 --- a/test/e2e/foyer_auth_test.go +++ b/test/e2e/foyer_auth_test.go @@ -548,13 +548,12 @@ var _ = Describe("krm-foyer", Label("foyer"), func() { }) // docs/ingress.md: krm-foyer terminates TLS itself by default, or sits behind a - // transparent ingress. These run with an nginx container in front of it. + // transparent ingress. The fixture has both kinds in front of it: Traefik (the + // Gateway, and the check's routes in foyer_check_test.go) and the nginx recipe + // (foyer_nginx_test.go), which also prove the shared origin, the login gate and its + // return path. These two remain. Context("behind an ingress", func() { PIt("takes its redirect URI and CSRF origin from configuration, whatever Host or X-Forwarded-Host says") PIt("delivers watch events without the ingress buffering them") - PIt("shares the origin with an application served by nginx on /, without seeing its requests") - PIt("sends a signed-out page load to login through /auth/check, and back to that page afterwards") - PIt("answers /auth/check with a status only: no token and no identity headers") - PIt("ignores a return path from the ingress header that is not a local path") }) }) diff --git a/test/e2e/foyer_check_test.go b/test/e2e/foyer_check_test.go new file mode 100644 index 0000000..c08d764 --- /dev/null +++ b/test/e2e/foyer_check_test.go @@ -0,0 +1,139 @@ +//go:build e2e + +package e2e + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/url" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// backendSaw is what the stand-in domain backend behind /auth/check?identity=true +// (hello-web-nginx.conf, /public/whoami) answers: the identity header it received, and +// the method. +type backendSaw struct { + Identity string `json:"identity"` + Method string `json:"method"` +} + +// checkIdentity is Krm-Foyer-Identity, decoded. +type checkIdentity struct { + UserInfo json.RawMessage `json:"userInfo"` + DisplayName string `json:"displayName"` + Connector string `json:"connector"` + Issuer string `json:"issuer"` + ExpiresAt string `json:"expiresAt"` +} + +// forgedIdentity is a Krm-Foyer-Identity a browser makes up: the API server's own +// administrator, from the audience connector. +var forgedIdentity = base64.RawURLEncoding.EncodeToString( + []byte(`{"userInfo":{"username":"system:admin","groups":["system:masters"]},"connector":"room-pass"}`)) + +// The routes of traefik-routes.yaml: Traefik's ForwardAuth asking /auth/check, as +// docs/ingress.md's Traefik recipe has it, through the real Traefik. +var _ = Describe("krm-foyer's check, behind Traefik", Label("foyer"), func() { + backend := func(ctx SpecContext, b *browser, method string, header http.Header) (answer, backendSaw) { + h := http.Header{"Krm-Foyer-Identity": {forgedIdentity}} + for k, v := range spoofedIdentity { + h[k] = v + } + for k, v := range header { + h[k] = v + } + a := b.do(ctx, method, "/public/whoami", nil, h) + var saw backendSaw + if a.Code == http.StatusOK { + Expect(json.Unmarshal(a.Body, &saw)).To(Succeed(), "%s", a.Body) + } + return a, saw + } + + It("tells a domain backend who the user is, as the API server says, whatever the browser forges", func(ctx SpecContext) { + b := fx.frontDoorBrowser() + + By("refusing a signed-out browser before the backend is reached") + a, _ := backend(ctx, b, http.MethodGet, nil) + Expect(a.Code).To(Equal(http.StatusUnauthorized), "%s", a.Body) + Expect(a.Header.Get("Krm-Foyer-Interruption")).To(Equal("Unauthorized")) + Expect(a.status().Kind).To(Equal("Status")) + + By("passing on the API server's answer for the signed-in user, never the browser's copy") + csrf := b.signedIn(ctx, alice) + a, saw := backend(ctx, b, http.MethodGet, nil) + // The backend's tripwire answers 400 to any cookie: a 200 means Traefik removed it. + Expect(a.Code).To(Equal(http.StatusOK), "%s", a.Body) + Expect(saw.Method).To(Equal(http.MethodGet)) + raw, err := base64.RawURLEncoding.DecodeString(saw.Identity) + Expect(err).NotTo(HaveOccurred(), "the backend received %q", saw.Identity) + var id checkIdentity + Expect(json.Unmarshal(raw, &id)).To(Succeed()) + Expect(string(raw)).NotTo(ContainSubstring("system:admin")) + Expect(string(raw)).NotTo(ContainSubstring("mallory")) + Expect(string(raw)).NotTo(MatchRegexp(jwtShape)) + Expect(id.DisplayName).To(Equal("alice")) + Expect(id.Issuer).To(Equal(fx.dexIssuer)) + + By("naming exactly whom /auth/whoami names") + who := b.get(ctx, "/auth/whoami") + Expect(who.Code).To(Equal(http.StatusOK), "%s", who.Body) + var whoami struct { + UserInfo json.RawMessage `json:"userInfo"` + } + Expect(json.Unmarshal(who.Body, &whoami)).To(Succeed()) + Expect(id.UserInfo).To(MatchJSON(whoami.UserInfo)) + var info whoamiAnswer + Expect(json.Unmarshal(who.Body, &info)).To(Succeed()) + Expect(info.UserInfo.Username).To(Equal(aliceK8sName)) + + By("holding a write to the backend to the rules of a write to /k8s") + a, _ = backend(ctx, b, http.MethodPost, http.Header{"Origin": {fx.foyerURL}}) + Expect(a.Code).To(Equal(http.StatusForbidden), "%s", a.Body) + Expect(a.status().Reason).To(Equal("CSRFProofRequired")) + a, _ = backend(ctx, b, http.MethodPost, http.Header{"Origin": {"https://evil.example"}, "X-Csrf-Token": {csrf}}) + Expect(a.Code).To(Equal(http.StatusForbidden), "%s", a.Body) + Expect(a.status().Reason).To(Equal("CrossOriginRequest")) + a, saw = backend(ctx, b, http.MethodPost, http.Header{"Origin": {fx.foyerURL}, "X-Csrf-Token": {csrf}}) + Expect(a.Code).To(Equal(http.StatusOK), "%s", a.Body) + Expect(saw.Method).To(Equal(http.MethodPost)) + Expect(saw.Identity).NotTo(BeEmpty()) + + By("refusing the session once it is signed out") + out := b.do(ctx, http.MethodPost, "/auth/logout", nil, http.Header{"Origin": {fx.foyerURL}, "X-Csrf-Token": {csrf}}) + Expect(out.Code).To(Equal(http.StatusNoContent)) + a, _ = backend(ctx, b, http.MethodGet, nil) + Expect(a.Code).To(Equal(http.StatusUnauthorized), "%s", a.Body) + }) + + It("sends a signed-out page load behind the login gate to the login, and back to that page", func(ctx SpecContext) { + b := fx.frontDoorBrowser() + page := http.Header{"Sec-Fetch-Mode": {"navigate"}, "Sec-Fetch-Dest": {"document"}} + + By("answering a script with the 401 it can act on") + a := b.do(ctx, http.MethodGet, "/members/", nil, http.Header{"Sec-Fetch-Mode": {"cors"}, "Sec-Fetch-Dest": {"empty"}}) + Expect(a.Code).To(Equal(http.StatusUnauthorized), "%s", a.Body) + + By("sending a page load to the login, which returns to the page") + a = b.do(ctx, http.MethodGet, "/members/", nil, page) + Expect(a.Code).To(Equal(http.StatusFound), "%s", a.Body) + // Absolute, at the public URL: Traefik would resolve a relative one against the + // check's address, krm-foyer's Service. + Expect(a.Header.Get("Location")).To(Equal(fx.foyerURL + "/auth/login?" + url.Values{"return_to": {"/members/"}}.Encode())) + start := b.get(ctx, a.Header.Get("Location")) + Expect(start.Code).To(Equal(http.StatusFound), "%s", start.Body) + authorize, err := url.Parse(start.Header.Get("Location")) + Expect(err).NotTo(HaveOccurred()) + done := b.get(ctx, b.atDex(ctx, authorize, alice)) + Expect(done.Code).To(Equal(http.StatusSeeOther)) + Expect(done.Header.Get("Location")).To(Equal("/members/")) + + By("serving the page once signed in, with no cookie past Traefik") + a = b.do(ctx, http.MethodGet, "/members/", nil, page) + Expect(a.Code).To(Equal(http.StatusOK), "%s", a.Body) + Expect(string(a.Body)).To(ContainSubstring("For signed-in members.")) + }) +}) diff --git a/test/e2e/foyer_nginx_test.go b/test/e2e/foyer_nginx_test.go new file mode 100644 index 0000000..b282110 --- /dev/null +++ b/test/e2e/foyer_nginx_test.go @@ -0,0 +1,84 @@ +//go:build e2e + +package e2e + +import ( + "net/http" + "net/url" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// docs/ingress.md's nginx recipe, as the document has it (front-door.sh copies it out +// of the text), in front of the main krm-foyer: a signed-out page load behind the gate +// goes to krm-foyer's login, and the login brings the browser back to the very page it +// asked for, whatever that page's query holds. +var _ = Describe("The nginx recipe", Label("foyer"), func() { + // signInThrough loads page through nginx signed out, follows the gate to krm-foyer's + // login and through Dex, and returns where the callback sends the browser. + signInThrough := func(ctx SpecContext, b *browser, page string) (login *url.URL, back string) { + GinkgoHelper() + gate := b.get(ctx, page) + Expect(gate.Code).To(Equal(http.StatusFound), "%s", gate.Body) + login, err := url.Parse(gate.Header.Get("Location")) + Expect(err).NotTo(HaveOccurred()) + Expect(login.Path).To(Equal("/auth/login")) + + started := b.get(ctx, login.String()) + Expect(started.Code).To(Equal(http.StatusFound), "krm-foyer refused the login the gate started: %s", started.Body) + authorize, err := url.Parse(started.Header.Get("Location")) + Expect(err).NotTo(HaveOccurred()) + callback := b.atDex(ctx, authorize, alice) + done := b.get(ctx, callback) + Expect(done.Code).To(Equal(http.StatusSeeOther), "%s", done.Body) + return login, done.Header.Get("Location") + } + + DescribeTable("returns to the whole page after login", + func(ctx SpecContext, page string) { + b := fx.nginxBrowser() + login, back := signInThrough(ctx, b, page) + Expect(back).To(Equal(page)) + Expect(login.Query()).To(Equal(url.Values{"return_to": {page}}), + "the gate's login link carries the page as one return_to, and nothing else") + + By("and the backend receives the page as the browser asked for it") + a := b.get(ctx, back) + Expect(a.Code).To(Equal(http.StatusOK), "%s", a.Body) + Expect(string(a.Body)).To(Equal("request=" + page + "\n")) + }, + Entry("with several query parameters", "/admin/edit?name=x&tab=history"), + Entry("with percent-encoding and a literal plus, each kept as sent", + "/admin/search?q=a%26b%3Dc%25&tag=x+y&sum=1%2B1"), + Entry("with a fragment-like and an equals sign encoded in a value", "/admin/edit?name=%23top&expr=a%3Db"), + // Inside the page's query, these are the page's own. They must not reach the + // login as krm-foyer's parameters, or as login parameters for the issuer. + Entry("with parameters named like the login's own", + "/admin/edit?return_to=%2Fevil&oidc.connector_id=other&name=x"), + Entry("with them unencoded", "/admin/edit?return_to=/evil&oidc.login_hint=mallory"), + ) + + It("leaves krm-foyer's local-path check in charge of where the browser goes", func(ctx SpecContext) { + b := fx.nginxBrowser() + // nginx matches //admin/x as /admin/x, and gates it; the page the browser asked + // for still starts with two slashes, which a browser reads as another host. + gate := b.get(ctx, "https://foyer.localhost:8443//admin/x") + Expect(gate.Code).To(Equal(http.StatusFound), "%s", gate.Body) + login := b.get(ctx, gate.Header.Get("Location")) + Expect(login.Code).To(Equal(http.StatusBadRequest), "%s", login.Body) + Expect(login.Header.Get("Location")).To(BeEmpty()) + }) + + It("turns a script's 401 into the login too, and passes a signed-in page and its identity to the backends", func(ctx SpecContext) { + b := fx.nginxBrowser() + a := b.do(ctx, http.MethodGet, "/admin/edit?name=x", nil, http.Header{"Sec-Fetch-Mode": {"cors"}}) + Expect(a.Code).To(Equal(http.StatusFound), "nginx turns every 401 from the check into the login: %s", a.Body) + + _, back := signInThrough(ctx, b, "/admin/") + Expect(back).To(Equal("/admin/")) + identity := b.get(ctx, "/public/whoami") + Expect(identity.Code).To(Equal(http.StatusOK), "%s", identity.Body) + Expect(string(identity.Body)).To(MatchRegexp(`(?m)^identity=[A-Za-z0-9_-]+$`)) + }) +}) diff --git a/test/e2e/foyer_rollout_test.go b/test/e2e/foyer_rollout_test.go new file mode 100644 index 0000000..bc31de5 --- /dev/null +++ b/test/e2e/foyer_rollout_test.go @@ -0,0 +1,111 @@ +//go:build e2e + +package e2e + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "sync" + "syscall" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Parked: a rollout that swallows no connection. docs/investigations/rollout-connections.md +// records what was measured and why this is not pursued yet; the roadmap keeps the item. +// +// A rollout replaces krm-foyer's one pod (the chart's Recreate), so between the two pods +// a new connection is refused, at once. What it should not do is leave a connection +// unanswered, during the rollout or after it. Today it does, now and then, at the +// moment the old pod's address goes away; the spec below shows it, and stays pending +// until a rollout without that is designed. The suite's own replacements forget the +// gone pod's connection tracking (fixture.forgetGonePods), which is what kept the +// rehearsal's 1,800 connections at once from timing out. +var _ = Describe("krm-foyer's rollout", Label("foyer"), func() { + PIt("refuses rather than swallows a connection while its pod is replaced, and leaves none to be swallowed afterwards", func(ctx SpecContext) { + // From a clean slate: nothing left over from pods replaced before. + fx.forgetGonePods() + // Two at a time, each a new connection that the client closes as soon as it is + // answered, until told to stop. Whether a connection is answered is the + // question; a request on it would add the server's own closing to the node's + // connection tracking. + var ( + mu sync.Mutex + failures []string + answered int + refused int + // A refusal can follow a retransmitted SYN: an answer all the same. + slowestRefusal time.Duration + ) + probing, stop := context.WithCancel(ctx) + var wg sync.WaitGroup + for range 2 { + wg.Go(func() { + for probing.Err() == nil { + started := time.Now() + c, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, "tcp", fx.foyerAddr) + mu.Lock() + switch { + case errors.Is(err, syscall.ECONNREFUSED): + refused++ + slowestRefusal = max(slowestRefusal, time.Since(started)) + case err != nil: + failures = append(failures, fmt.Sprintf("%s: %v", started.Format("15:04:05.000"), err)) + default: + answered++ + _ = c.Close() + } + mu.Unlock() + time.Sleep(20 * time.Millisecond) + } + }) + } + + pods := func() string { + return fx.kubectl("-n", fx.foyerNamespace, "get", "pods", "-l", "app.kubernetes.io/instance=krm-foyer", + "-o", `jsonpath={range .items[*]}{.metadata.name}={.status.phase}{" "}{end}`) + } + old := pods() + fx.kubectl("-n", fx.foyerNamespace, "rollout", "restart", "deployment/krm-foyer") + fx.kubectl("-n", fx.foyerNamespace, "rollout", "status", "deployment/krm-foyer", "--timeout=120s") + eventually(ctx, func() error { + now := strings.Fields(pods()) + if len(now) != 1 || strings.Contains(old, now[0]) { + return fmt.Errorf("pods %q, before %q", now, old) + } + return nil + }).WithTimeout(2 * time.Minute).Should(Succeed()) + // And a little past the old pod's end, when an address that is gone would show. + time.Sleep(3 * time.Second) + stop() + wg.Wait() + fx.forgetGonePods() + Expect(answered).To(BeNumerically(">", 100), "too few probes to say anything") + Expect(failures).To(BeEmpty(), "%d connections answered, %d refused", answered, refused) + AddReportEntry("rollout", fmt.Sprintf("%d connections answered, %d refused while no pod was ready, the slowest refusal after %v", + answered, refused, slowestRefusal.Round(time.Millisecond))) + + By("then opening 1,800 connections at once, as the rehearsal does: none is swallowed") + var dialFailures []string + var conns []net.Conn + inParallel(1800, 1800, func(int) { + c, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(ctx, "tcp", fx.foyerAddr) + mu.Lock() + defer mu.Unlock() + if err != nil { + dialFailures = append(dialFailures, err.Error()) + return + } + conns = append(conns, c) + }) + for _, c := range conns { + _ = c.Close() + } + Expect(dialFailures).To(BeEmpty()) + }, SpecTimeout(4*time.Minute)) +}) diff --git a/test/e2e/foyer_scope_test.go b/test/e2e/foyer_scope_test.go new file mode 100644 index 0000000..4c2671f --- /dev/null +++ b/test/e2e/foyer_scope_test.go @@ -0,0 +1,156 @@ +//go:build e2e + +package e2e + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// A browser identity, the first step of docs/application-scope.md: the cluster's own +// authentication configuration names one person differently by the client the token +// was issued to, so an operator's broad grants go to their command-line name and never +// reach a script on an application's origin. Nothing in krm-foyer is involved; this +// proves the recipe the document gives. +var _ = Describe("A browser identity", Label("foyer"), func() { + It("names the same person differently on the command line, and keeps that name's grants out of krm-foyer's reach", func(ctx SpecContext) { + cli := fx.login(ctx, alice, cliClient, cliSecret) + code, name := fx.selfSubjectReview(ctx, cli) + Expect(code).To(Equal(http.StatusCreated)) + Expect(name).To(Equal("kubectl:" + alice)) + + u := signIn(ctx, alice) + code, name = fx.selfSubjectReview(ctx, u.token) + Expect(code).To(Equal(http.StatusCreated)) + Expect(name).To(Equal(aliceK8sName)) + + By("granting the command-line name, as an operator's admin grant would be") + ns := fx.namespace() + fx.grant(ns, "kubectl:"+alice, "secrets", "list") + secrets := "/api/v1/namespaces/" + ns + "/secrets" + Expect(fx.direct(ctx, cli, http.MethodGet, secrets, nil).Code).To(Equal(http.StatusOK)) + + By("refusing the same person through krm-foyer, as the API server would refuse oidc:alice") + a := u.viaFoyer(ctx, http.MethodGet, secrets, nil, nil) + Expect(a.Code).To(Equal(http.StatusForbidden), "%s", a.Body) + Expect(a.status().Message).To(ContainSubstring(`User "` + aliceK8sName + `"`)) + Expect(fx.direct(ctx, u.token, http.MethodGet, secrets, nil).Code).To(Equal(http.StatusForbidden)) + + By("keeping krm-foyer's name when krm-foyer asks Dex for the command line's audience too") + // Dex's cross-client scope, as krm-foyer asks for the cluster's audience in a + // deployment like Voter's: Dex adds the asking client to the audiences and makes + // it azp. + cross := fx.loginScopes(ctx, alice, foyerClient, foyerSecret, "openid email profile audience:server:client_id:"+cliClient) + Expect(tokenClaims(cross)).To(And(HaveKeyWithValue("aud", ConsistOf(cliClient, foyerClient)), HaveKeyWithValue("azp", foyerClient))) + code, name = fx.selfSubjectReview(ctx, cross) + Expect(code).To(Equal(http.StatusCreated)) + Expect(name).To(Equal(aliceK8sName)) + Expect(fx.direct(ctx, cross, http.MethodGet, secrets, nil).Code).To(Equal(http.StatusForbidden)) + + By("still refusing a client the cluster does not accept") + code, _ = fx.selfSubjectReview(ctx, fx.login(ctx, alice, otherClient, otherSecret)) + Expect(code).To(Equal(http.StatusUnauthorized)) + }) + + // Every form a token's client can take, minted by the test issuer, which the API + // server holds to Dex's rules. The client is azp when there is one, and then it must + // be one of the audiences; without azp it is the audience, a string or a list of + // one. Only krm-foyer and kubectl are clients, and anything else, or any doubt + // about which client it is, is no identity at all: never the command line's name, + // whose grants are the broad ones. + Describe("by the token's client", Ordered, ContinueOnFailure, func() { + const carol = "carol@example.com" + var secrets string + + BeforeAll(func(ctx SpecContext) { + ns := fx.namespace() + secrets = "/api/v1/namespaces/" + ns + "/secrets" + // The command line's grants, to its name and to one of its groups. + fx.grant(ns, "kubectl:"+carol, "secrets", "get") + fx.grantGroup(ns, "kubectl:admins", "secrets", "list") + cli := mintFor(map[string]any{"aud": cliClient}) + eventually(ctx, func() int { return fx.direct(ctx, cli, http.MethodGet, secrets, nil).Code }). + Should(Equal(http.StatusOK), "the group grant must work, or the refusals below prove nothing") + }) + + DescribeTable("names a token by its client", + func(ctx SpecContext, claims map[string]any, prefix string) { + token := mintFor(claims) + code, name, groups := fx.selfSubjectReviewGroups(ctx, token) + Expect(code).To(Equal(http.StatusCreated)) + Expect(name).To(Equal(prefix + carol)) + Expect(groups).To(ContainElement(prefix + "admins")) + + listed := fx.direct(ctx, token, http.MethodGet, secrets, nil).Code + if prefix == "kubectl:" { + Expect(listed).To(Equal(http.StatusOK)) + } else { + Expect(listed).To(Equal(http.StatusForbidden), "the command line's group grant reached a browser token") + } + }, + Entry("krm-foyer's audience", map[string]any{"aud": foyerClient}, "oidc:"), + Entry("krm-foyer's audience as a list of one", map[string]any{"aud": []string{foyerClient}}, "oidc:"), + Entry("krm-foyer's audience, and krm-foyer as azp", map[string]any{"aud": foyerClient, "azp": foyerClient}, "oidc:"), + // Dex's cross-client scope: krm-foyer asked for another client's audience. + Entry("krm-foyer with the command line's audience too, as azp", + map[string]any{"aud": []string{cliClient, foyerClient}, "azp": foyerClient}, "oidc:"), + Entry("the command line's audience", map[string]any{"aud": cliClient}, "kubectl:"), + Entry("the command line's audience as a list of one", map[string]any{"aud": []string{cliClient}}, "kubectl:"), + Entry("the command line with krm-foyer's audience too, as azp", + map[string]any{"aud": []string{foyerClient, cliClient}, "azp": cliClient}, "kubectl:"), + ) + + DescribeTable("refuses a token whose client is unknown or in doubt, and with it every grant", + func(ctx SpecContext, claims map[string]any) { + token := mintFor(claims) + code, name, groups := fx.selfSubjectReviewGroups(ctx, token) + Expect(code).To(Equal(http.StatusUnauthorized), "named %q, groups %q", name, groups) + Expect(name).To(BeEmpty()) + Expect(fx.direct(ctx, token, http.MethodGet, secrets, nil).Code).To(Equal(http.StatusUnauthorized)) + Expect(fx.direct(ctx, token, http.MethodGet, secrets+"/any", nil).Code).To(Equal(http.StatusUnauthorized)) + }, + Entry("azp naming a client that is not an audience", + map[string]any{"aud": foyerClient, "azp": "another-browser"}), + Entry("azp naming krm-foyer, which is not an audience", + map[string]any{"aud": cliClient, "azp": foyerClient}), + Entry("azp naming another client among krm-foyer's audiences", + map[string]any{"aud": []string{foyerClient, otherClient}, "azp": otherClient}), + // What Dex issues another client that lists kubectl as a trusted peer. + Entry("azp naming another client given the command line's audience", + map[string]any{"aud": []string{cliClient, otherClient}, "azp": otherClient}), + Entry("two audiences and no azp", map[string]any{"aud": []string{foyerClient, cliClient}}), + Entry("two audiences the other way round and no azp", map[string]any{"aud": []string{cliClient, foyerClient}}), + Entry("azp as a list", map[string]any{"aud": foyerClient, "azp": []string{foyerClient}}), + Entry("azp as a number", map[string]any{"aud": foyerClient, "azp": 7}), + Entry("azp as null", map[string]any{"aud": foyerClient, "azp": nil}), + Entry("azp empty", map[string]any{"aud": foyerClient, "azp": ""}), + Entry("another client's audience", map[string]any{"aud": otherClient}), + ) + }) +}) + +// tokenClaims decodes a token's claims, unverified, to see what an issuer put in it. +func tokenClaims(token string) map[string]any { + GinkgoHelper() + parts := strings.Split(token, ".") + Expect(parts).To(HaveLen(3)) + raw, err := base64.RawURLEncoding.DecodeString(parts[1]) + Expect(err).NotTo(HaveOccurred()) + var claims map[string]any + Expect(json.Unmarshal(raw, &claims)).To(Succeed()) + return claims +} + +// mintFor mints a verified token for carol in the group admins, with claims on top. +func mintFor(claims map[string]any) string { + all := map[string]any{"email": "carol@example.com", "email_verified": true, "groups": []string{"admins"}} + for k, v := range claims { + all[k] = v + } + return fx.mint(all) +} diff --git a/test/e2e/hello_test.go b/test/e2e/hello_test.go index f4502ab..09a2c9a 100644 --- a/test/e2e/hello_test.go +++ b/test/e2e/hello_test.go @@ -275,14 +275,18 @@ var _ = Describe("The hello example", Label("browser"), Ordered, func() { }) // startBrowser starts Chromium in this container's network namespace and returns a tab -// in it. Both are gone when the spec tree is done. -func startBrowser(ctx context.Context) context.Context { +// in it. Both are gone when the spec tree is done. It trusts the front door's and Dex's +// certificates, and those in the files more names. +func startBrowser(ctx context.Context, more ...string) context.Context { // Trust exactly the certificates of the front door and Dex, by their public keys: // the fixture's CA is not in the browser's store. trusted := []string{ spki(filepath.Join(fx.dir, "foyer", "tls.crt")), spki(filepath.Join(fx.dir, "tls", "dex.crt")), } + for _, file := range more { + trusted = append(trusted, spki(file)) + } // This container: the devcontainer, or the CI job's container. Docker names it by // its hostname unless told otherwise. self, err := os.Hostname() diff --git a/test/e2e/room_pass_test.go b/test/e2e/room_pass_test.go new file mode 100644 index 0000000..0eb10c8 --- /dev/null +++ b/test/e2e/room_pass_test.go @@ -0,0 +1,600 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "net" + "net/http" + "net/http/cookiejar" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// Room Pass's QR login through krm-foyer (docs/room-pass.md), in a real browser, against +// Room Pass 2.0.0 and the Dex behind it (test/e2e/cluster/room-pass.sh): a participant +// scans the code on the presenter's screen, types a display name, and is signed in to +// the application through krm-foyer, with Kubernetes naming them as Room Pass's rules +// say. Three programs share the application's host, room.localhost: krm-foyer-room, +// Room Pass's /join, and the test application with the QR entry point. If their routes +// or the join cookie go wrong, the flow does not fail: Room Pass quietly asks for the +// code again. Only a browser shows that, so this spec checks the join page itself. +var _ = Describe("Room Pass's QR login", Label("browser", "room-pass"), Ordered, func() { + var ( + room roomFixture + tab context.Context + started time.Time + ns string + code, display, email string + seen *seenTraffic + name string + ) + + BeforeAll(func(ctx SpecContext) { + room = loadRoom() + started = time.Now() + tab = startBrowser(ctx, filepath.Join(fx.dir, "room", "edge.crt")) + seen = watchTraffic(tab) + ns = fx.namespace() + // The Room's audience group may write ConfigMaps here: the participant's grant. + fx.grantGroup(ns, room.audienceGroup, "configmaps", "get", "create") + // A name nobody in the Room has taken: the name is the participant's identity. + name = "Ada Lovelace " + randomID() + display = strings.ReplaceAll(name, " ", "-") + email = strings.ToLower(display) + "@koudijs.dev.test" + }, NodeTimeout(2*time.Minute)) + + It("arrives at Room Pass with the scanned code already supplied", func() { + code = room.currentCode() + // From a link on another site, as a scanner app or a chat hands the URL on. The + // redirects after it are then cross-site, which is what SameSite=Lax is for: + // typed into the address bar, even a Strict cookie would come along. + follow(tab, fx.foyerURL+"/", room.url+"/join-room?code="+code) + + // QR -> /join-room (the application) -> krm-foyer's login -> Dex -> Room Pass's + // callback, /bind and confirm -> /join, back on the application's host. + Expect(atJoinPage(tab)).To(HavePrefix(room.url + "/join?handoff=")) + pill, field := joinPage(tab) + Expect(field).To(BeFalse(), "Room Pass asked for the room code: the join cookie did not reach /join") + Expect(pill).To(Equal(code)) + + By("having kept the code out of every URL but the QR code's own, and out of the authorization request") + for _, u := range seen.urls() { + if strings.HasPrefix(u, room.url+"/join-room?") { + continue + } + Expect(u).NotTo(ContainSubstring(code), "the code travelled in a URL") + } + Expect(seen.urls()).To(ContainElement(HavePrefix(room.issuer+"/auth?")), "the login went through Room Pass's Dex") + Expect(seen.urls()).To(ContainElement(ContainSubstring("connector_id=room-pass"))) + }) + + It("signs the participant in with only a display name, and returns to the application's page", func() { + run(tab, + chromedp.SendKeys("#rp-name", name, chromedp.ByID), + chromedp.Click(`form[action="/join"] button`), + ) + Eventually(func() string { + var location string + run(tab, chromedp.Location(&location)) + return location + }).WithTimeout(20 * time.Second).Should(Equal(room.url + "/room/")) + var heading string + run(tab, chromedp.Text("p", &heading)) + Expect(heading).To(Equal("You are in the room.")) + }) + + It("shows connector room-pass and the display name in /auth/session, and keeps every credential from the page", func() { + s := pageSession(tab) + Expect(s.Authenticated).To(BeTrue()) + Expect(s.Issuer).To(Equal(room.issuer)) + Expect(s.Connector).To(Equal("room-pass")) + Expect(s.DisplayName).To(Equal(display), "Room Pass stores the name folded, as it was shown on the join page") + Expect(s.Email).To(Equal(email)) + Expect(s.Groups).To(Equal([]string{room.audienceGroup})) + + var cookies, storage string + run(tab, + chromedp.Evaluate(`document.cookie`, &cookies), + chromedp.Evaluate(`JSON.stringify([localStorage, sessionStorage])`, &storage), + ) + Expect(cookies).To(BeEmpty(), "every cookie on the application's host must be HttpOnly") + Expect(storage).To(Equal("[{},{}]")) + Expect(inPage(tab, `fetch('/auth/session').then(r => r.text())`)).NotTo(MatchRegexp(jwtShape)) + }) + + var username string + It("is the participant to Kubernetes, with both attribution extras on a permitted write", func() { + var answer struct { + UserInfo struct { + Username string `json:"username"` + Groups []string `json:"groups"` + Extra map[string][]string `json:"extra"` + } `json:"userInfo"` + } + whoami := inPage(tab, `fetch('/auth/whoami').then(r => r.text())`) + Expect(whoami).NotTo(MatchRegexp(jwtShape)) + Expect(json.Unmarshal([]byte(whoami), &answer)).To(Succeed(), whoami) + who := answer.UserInfo + // Dex's opaque subject, never the name the participant typed. + Expect(who.Username).To(HavePrefix("demo:")) + Expect(who.Username).NotTo(ContainSubstring(display)) + Expect(who.Groups).To(ContainElement(room.audienceGroup)) + Expect(who.Extra).To(HaveKeyWithValue("configbutler.ai/claims/display-name", []string{display})) + Expect(who.Extra).To(HaveKeyWithValue("configbutler.ai/claims/email", []string{email})) + username = who.Username + + cm := "room-" + randomID() + created := pageWrite(tab, "/k8s/api/v1/namespaces/"+ns+"/configmaps", string(configMap(cm))) + Expect(created.Status).To(Equal(http.StatusCreated), created.Body) + Expect(fx.kubectl("-n", ns, "get", "configmap", cm, "-o", "name")).To(Equal("configmap/" + cm)) + + var write auditEvent + eventually(context.Background(), func() []auditEvent { + return slices.DeleteFunc(fx.audited(cm), func(e auditEvent) bool { return e.Verb != "create" }) + }).Should(HaveLen(1)) + write = slices.DeleteFunc(fx.audited(cm), func(e auditEvent) bool { return e.Verb != "create" })[0] + Expect(write.ResponseStatus.Code).To(Equal(http.StatusCreated)) + Expect(write.User.Username).To(Equal(username)) + Expect(write.ImpersonatedUser).To(BeNil()) + Expect(write.User.Extra).To(HaveKeyWithValue("configbutler.ai/claims/display-name", []string{display})) + Expect(write.User.Extra).To(HaveKeyWithValue("configbutler.ai/claims/email", []string{email})) + + By("and nothing the group was not granted") + refused := pageWrite(tab, "/k8s/api/v1/namespaces/"+ns+"/secrets", `{"apiVersion":"v1","kind":"Secret","metadata":{"name":"no"}}`) + Expect(refused.Status).To(Equal(http.StatusForbidden), refused.Body) + Expect(refused.Body).To(ContainSubstring(`User \"` + username + `\"`)) + }) + + It("keeps the session when krm-foyer's pod is replaced", func(ctx SpecContext) { + room.replaceFoyer(ctx) + Eventually(func() int { + return inPageStatus(tab, "/auth/whoami") + }).WithTimeout(time.Minute).WithPolling(time.Second).Should(Equal(http.StatusOK)) + s := pageSession(tab) + Expect(s.Authenticated).To(BeTrue()) + Expect(s.Connector).To(Equal("room-pass")) + Expect(s.DisplayName).To(Equal(display)) + created := pageWrite(tab, "/k8s/api/v1/namespaces/"+ns+"/configmaps", string(configMap("after-"+randomID()))) + Expect(created.Status).To(Equal(http.StatusCreated), created.Body) + }, SpecTimeout(3*time.Minute)) + + It("signs out of krm-foyer, then of Room Pass, each by its own contract", func() { + // A copy of the session cookie, taken as anyone who read it could. Logout clears + // the browser's cookie; it revokes nothing (docs/design.md, "Sessions"). + copied := cookieNamed(tab, room.url, sessionCookie) + Expect(copied).NotTo(BeEmpty()) + + By("krm-foyer: POST /auth/logout with the session's CSRF proof") + Expect(inPage(tab, `fetch('/auth/session').then(r => r.json()).then(s => + fetch('/auth/logout', {method: 'POST', headers: {[s.csrfHeader]: s.csrfToken}})).then(r => String(r.status))`)). + To(Equal("204")) + Expect(inPageStatus(tab, "/auth/session")).To(Equal(http.StatusUnauthorized)) + Expect(cookieNamed(tab, room.url, sessionCookie)).To(BeEmpty()) + + By("which is not a revocation: the copy taken before is still a session until it expires") + Expect(room.sessionWith(copied)).To(Equal(http.StatusOK)) + + By("Room Pass: its sign-out form on /join, a POST with its own CSRF proof") + run(tab, chromedp.Navigate(room.url+"/join"), chromedp.WaitVisible(`form[action="/logout"] button`)) + var welcome string + run(tab, chromedp.Text("body", &welcome)) + Expect(welcome).To(ContainSubstring("You’re already enrolled as " + display)) + Expect(cookieNamed(tab, room.url, "__Host-rp-session")).NotTo(BeEmpty()) + run(tab, chromedp.Click(`form[action="/logout"] button`)) + Eventually(func() bool { + _, field := joinPage(tab) + return field + }).WithTimeout(20*time.Second).Should(BeTrue(), "after Room Pass's sign-out, /join asks for a code again") + Expect(cookieNamed(tab, room.url, "__Host-rp-session")).To(BeEmpty()) + + By("so a new scan asks for a name again: this browser is no participant any more") + follow(tab, fx.foyerURL+"/", room.url+"/join-room?code="+room.currentCode()) + atJoinPage(tab) + var page string + run(tab, chromedp.Text("body", &page)) + Expect(page).To(ContainSubstring("Choose a display name to join.")) + Expect(page).NotTo(ContainSubstring("already enrolled")) + }) + + // The failure this spec exists for, made on purpose: without the join cookie on the + // application's host, Room Pass asks for the code, and the check above notices. + It("tells the silent fallback apart: without the join cookie, Room Pass asks for the code", func() { + run(tab, chromedp.Navigate(room.url+"/auth/login?return_to=%2Froom%2F&oidc.connector_id=room-pass")) + atJoinPage(tab) + pill, field := joinPage(tab) + Expect(pill).To(BeEmpty()) + Expect(field).To(BeTrue()) + }) + + It("leaves no credential in a response the browser saw, or in the logs of the edge, krm-foyer and Room Pass", func() { + Expect(seen.leaks()).To(BeEmpty()) + for _, logs := range []struct{ ns, deployment string }{ + {fx.foyerNamespace, room.foyerRelease}, {room.namespace, "room-pass"}, {"fixture", "room-app"}, + {"traefik-system", "traefik"}, + } { + text := fx.kubectl("-n", logs.ns, "logs", "deployment/"+logs.deployment, + "--since-time="+started.UTC().Format(time.RFC3339), "--tail=-1") + Expect(text).NotTo(ContainSubstring("/join-room?code="), "the QR URL in the log of %s", logs.deployment) + Expect(text).NotTo(MatchRegexp(jwtShape), "the log of %s", logs.deployment) + Expect(text).NotTo(ContainSubstring(code), "the join code in the log of %s", logs.deployment) + for _, c := range seen.cookieValues() { + Expect(text).NotTo(ContainSubstring(c), "a cookie in the log of %s", logs.deployment) + } + } + }) +}) + +// Dex's authproxy connector believes the X-Remote-* headers on its callback. Room Pass +// is the only thing that may set them: the edge strips them, Room Pass strips them, and +// a NetworkPolicy keeps everything but Room Pass from Dex's port. +var _ = Describe("Room Pass's Dex", Label("room-pass"), func() { + It("is reachable only from Room Pass, so nothing else can forge the headers it believes", func(ctx SpecContext) { + room := loadRoom() + dex := "http://dex." + room.namespace + ".svc:5556/.well-known/openid-configuration" + + By("from a pod labelled as Room Pass, in its namespace: the policy's one way in") + out, err := room.probe(ctx, room.namespace, "app=room-pass", dex) + Expect(err).NotTo(HaveOccurred(), "%s", out) + Expect(out).To(MatchRegexp(`"issuer":\s*"` + regexp.QuoteMeta(room.issuer) + `"`)) + + By("from any other pod, here or in another namespace: no connection at all") + // k3s's policy controller rejects the connection rather than dropping it. The + // pod above reached the same Service a moment ago, so the refusal is the policy's. + for _, from := range []struct{ ns, labels string }{ + {room.namespace, "app=not-room-pass"}, {"fixture", "app=room-pass"}, + } { + out, err := room.probe(ctx, from.ns, from.labels, dex) + Expect(err).To(HaveOccurred(), "a pod in %s with %s reached Dex: %s", from.ns, from.labels, out) + Expect(out).To(ContainSubstring("can't connect to remote host"), "%s", out) + } + }) + + It("never takes an identity from a browser's headers, even on Room Pass's own paths", func(ctx SpecContext) { + room := loadRoom() + c := room.client() + // A browser's cookie jar: Room Pass binds a login to the browser that started it. + jar, err := cookiejar.New(nil) + Expect(err).NotTo(HaveOccurred()) + c.Jar = jar + forged := http.Header{ + "X-Remote-User": {"mallory"}, "X-Remote-User-Id": {"mallory"}, "X-Remote-User-Name": {"mallory"}, + "X-Remote-User-Email": {"mallory@example.com"}, "X-Remote-Group": {"demo:krm-foyer-e2e"}, + } + get := func(target string) *http.Response { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + Expect(err).NotTo(HaveOccurred()) + for k, v := range forged { + req.Header[k] = v + } + resp, err := c.Do(req) + Expect(err).NotTo(HaveOccurred()) + _ = resp.Body.Close() + return resp + } + + By("following a login from krm-foyer with the headers on every request: it ends at the join form") + next := room.url + "/auth/login?return_to=%2Froom%2F&oidc.connector_id=room-pass" + for range 10 { + resp := get(next) + if resp.StatusCode == http.StatusOK { + break + } + Expect(resp.StatusCode).To(BeNumerically(">=", 300), "%s answered %d", next, resp.StatusCode) + Expect(resp.StatusCode).To(BeNumerically("<", 400), "%s answered %d", next, resp.StatusCode) + loc, err := resp.Request.URL.Parse(resp.Header.Get("Location")) + Expect(err).NotTo(HaveOccurred()) + Expect(loc.Path).NotTo(Equal("/auth/callback"), "Dex issued a code for headers a browser sent") + next = loc.String() + } + Expect(next).To(HavePrefix(room.url + "/join?handoff=")) + + By("and Room Pass's callback and completion refuse them outright") + Expect(get(room.issuer + "/callback/room-pass").StatusCode).To(Equal(http.StatusBadRequest)) + Expect(get(room.issuer + "/room-pass/complete?handoff=x").StatusCode).To(Equal(http.StatusForbidden)) + }) +}) + +// roomFixture is what room-pass.sh left behind (E2E_DIR/room-env). +type roomFixture struct { + url, issuer, namespace, name, audienceGroup, foyerRelease string +} + +func loadRoom() roomFixture { + GinkgoHelper() + env := readEnv(filepath.Join(fx.dir, "room-env"), "Room Pass is not deployed; run `task e2e-deploy` (or `task test-e2e`)") + return roomFixture{ + url: env["ROOM_URL"], issuer: env["ROOM_PASS_ISSUER"], namespace: env["ROOM_PASS_NAMESPACE"], + name: env["ROOM_NAME"], audienceGroup: env["ROOM_AUDIENCE_GROUP"], foyerRelease: env["ROOM_FOYER_RELEASE"], + } +} + +// currentCode is the Room's newest join code, as Room Pass's presenter tool reads it: +// from the Room's status, with an operator's credential. +func (r roomFixture) currentCode() string { + GinkgoHelper() + var status struct { + Status struct { + ValidJoinCodes []struct { + Code string `json:"code"` + IssuedAt time.Time `json:"issuedAt"` + } `json:"validJoinCodes"` + } `json:"status"` + } + Expect(json.Unmarshal([]byte(fx.kubectl("-n", r.namespace, "get", "room", r.name, "-o", "json")), &status)).To(Succeed()) + codes := status.Status.ValidJoinCodes + Expect(codes).NotTo(BeEmpty()) + newest := slices.MaxFunc(codes, func(a, b struct { + Code string `json:"code"` + IssuedAt time.Time `json:"issuedAt"` + }) int { + return a.IssuedAt.Compare(b.IssuedAt) + }) + return newest.Code +} + +// replaceFoyer restarts krm-foyer-room and returns once its new pod is the only one. +func (r roomFixture) replaceFoyer(ctx context.Context) { + GinkgoHelper() + selector := "app.kubernetes.io/instance=" + r.foyerRelease + pods := func() string { + return fx.kubectl("-n", fx.foyerNamespace, "get", "pods", "-l", selector, + "-o", `jsonpath={range .items[*]}{.metadata.name}={.status.phase}{" "}{end}`) + } + old := pods() + fx.kubectl("-n", fx.foyerNamespace, "rollout", "restart", "deployment/"+r.foyerRelease) + fx.kubectl("-n", fx.foyerNamespace, "rollout", "status", "deployment/"+r.foyerRelease, "--timeout=120s") + eventually(ctx, func() error { + now := strings.Fields(pods()) + if len(now) != 1 || !strings.HasSuffix(now[0], "=Running") || strings.Contains(old, now[0]) { + return fmt.Errorf("pods %q, before %q", now, old) + } + return nil + }).WithTimeout(2 * time.Minute).Should(Succeed()) +} + +// client reaches both of the Room's hosts through the front door's port-forward, as a +// browser does, and follows no redirect. +func (r roomFixture) client() *http.Client { + GinkgoHelper() + roots := x509.NewCertPool() + ca, err := os.ReadFile(filepath.Join(fx.dir, "ca.crt")) + Expect(err).NotTo(HaveOccurred()) + Expect(roots.AppendCertsFromPEM(ca)).To(BeTrue()) + dialer := &net.Dialer{Timeout: 5 * time.Second} + return &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + _, port, _ := net.SplitHostPort(addr) + return dialer.DialContext(ctx, network, net.JoinHostPort("127.0.0.1", port)) + }, + }, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } +} + +// sessionWith asks krm-foyer-room for /auth/session with a session cookie of this value. +func (r roomFixture) sessionWith(cookie string) int { + GinkgoHelper() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, r.url+"/auth/session", nil) + Expect(err).NotTo(HaveOccurred()) + req.AddCookie(&http.Cookie{Name: sessionCookie, Value: cookie}) + resp, err := r.client().Do(req) + Expect(err).NotTo(HaveOccurred()) + _ = resp.Body.Close() + return resp.StatusCode +} + +// probe fetches target from a short-lived pod in ns with labels, waiting at most five +// seconds, and returns what it printed. The pod is never ready, so no Service selecting +// its labels sends it traffic. +func (r roomFixture) probe(ctx context.Context, ns, labels, target string) (string, error) { + GinkgoHelper() + pod := "probe-" + randomID() + overrides := `{"spec":{"automountServiceAccountToken":false,"containers":[{"name":"` + pod + `","image":"` + busyboxImage + + `","args":["wget","-q","-T","5","-O","-",` + strconv.Quote(target) + `],` + + `"readinessProbe":{"exec":{"command":["false"]}},` + + `"securityContext":{"runAsNonRoot":true,"runAsUser":65534,"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}}]}}` + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + out, err := exec.CommandContext(ctx, "kubectl", "--kubeconfig", filepath.Join(fx.dir, "kubeconfig"), + "-n", ns, "run", pod, "--rm", "-i", "--quiet", "--restart=Never", + "--image="+busyboxImage, "--labels="+labels, "--overrides="+overrides).CombinedOutput() + return string(out), err +} + +// busyboxImage is start-cluster.sh's busybox. +const busyboxImage = "busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" + +// follow opens page, then a link on it to target: a navigation that another site +// started. +func follow(tab context.Context, page, target string) { + GinkgoHelper() + run(tab, chromedp.Navigate(page), chromedp.Evaluate(`{ + const a = document.createElement('a'); + a.href = `+strconv.Quote(target)+`; + document.body.append(a); + a.click(); + }`, nil)) +} + +// atJoinPage waits for Room Pass's join form and returns its URL. If it never comes, +// it fails with where the browser is instead, and what that page says. +func atJoinPage(tab context.Context) string { + GinkgoHelper() + var location, text string + Eventually(func() bool { + ctx, cancel := context.WithTimeout(tab, 5*time.Second) + defer cancel() + var form bool + err := chromedp.Run(ctx, + chromedp.Location(&location), + chromedp.Evaluate(`document.body ? document.body.innerText : ''`, &text), + chromedp.Evaluate(`document.readyState === 'complete' && !!document.getElementById('rp-name')`, &form), + ) + return err == nil && form + }).WithTimeout(20*time.Second).WithPolling(250*time.Millisecond).Should(BeTrue(), + "no join form; the browser is at %s, which says:\n%.600s", location, text) + return location +} + +// joinPage reads Room Pass's join form: the code the page says was scanned, and +// whether it shows a field to type a code into instead. +func joinPage(tab context.Context) (scanned string, codeField bool) { + GinkgoHelper() + var state struct { + Scanned string `json:"scanned"` + Field bool `json:"field"` + } + raw := inPage(tab, `JSON.stringify({ + scanned: document.querySelector('.scanned strong')?.textContent ?? '', + field: [...document.querySelectorAll('input[name="code"]')].some(i => i.type !== 'hidden'), + })`) + Expect(json.Unmarshal([]byte(raw), &state)).To(Succeed(), raw) + return state.Scanned, state.Field +} + +// roomSession is /auth/session's answer with the session claims. +type roomSession struct { + sessionState + DisplayName string `json:"displayName"` + Groups []string `json:"groups"` + Connector string `json:"connector"` +} + +func pageSession(tab context.Context) roomSession { + GinkgoHelper() + raw := inPage(tab, `fetch('/auth/session').then(r => r.text())`) + var s roomSession + Expect(json.Unmarshal([]byte(raw), &s)).To(Succeed(), raw) + return s +} + +// inPage evaluates script in the tab's page and returns its string result. +func inPage(tab context.Context, script string) string { + GinkgoHelper() + var out string + run(tab, chromedp.Evaluate(script, &out, awaitPromise)) + return out +} + +// inPageStatus is the status of a GET of path from the page, or 0 if it failed. +func inPageStatus(tab context.Context, path string) int { + GinkgoHelper() + n, err := strconv.Atoi(inPage(tab, `fetch(`+strconv.Quote(path)+`).then(r => String(r.status), () => "0")`)) + Expect(err).NotTo(HaveOccurred()) + return n +} + +// pageWrite POSTs body to path from the page, as the application would: with the +// session's CSRF proof, which only a page on the origin can read. +func pageWrite(tab context.Context, path, body string) (a struct { + Status int `json:"status"` + Body string `json:"body"` +}, +) { + GinkgoHelper() + raw := inPage(tab, `fetch('/auth/session').then(r => r.json()).then(s => fetch(`+strconv.Quote(path)+`, { + method: 'POST', + headers: {'Content-Type': 'application/json', [s.csrfHeader]: s.csrfToken}, + body: `+strconv.Quote(body)+`, + })).then(async r => JSON.stringify({status: r.status, body: await r.text()}))`) + Expect(json.Unmarshal([]byte(raw), &a)).To(Succeed(), raw) + return a +} + +// cookieNamed is the browser's cookie name for target, HttpOnly or not, or "". +func cookieNamed(tab context.Context, target, name string) string { + GinkgoHelper() + var cookies []*network.Cookie + run(tab, chromedp.ActionFunc(func(ctx context.Context) error { + var err error + cookies, err = network.GetCookies().WithURLs([]string{target}).Do(ctx) + return err + })) + for _, c := range cookies { + if c.Name == name { + return c.Value + } + } + return "" +} + +// seenTraffic is every request URL and response header the browser saw, from the +// DevTools protocol, for the credential scan. +type seenTraffic struct { + mu sync.Mutex + urlList []string + headers []string + cookies []string +} + +func watchTraffic(tab context.Context) *seenTraffic { + s := &seenTraffic{} + chromedp.ListenTarget(tab, func(ev any) { + s.mu.Lock() + defer s.mu.Unlock() + switch e := ev.(type) { + case *network.EventRequestWillBeSent: + s.urlList = append(s.urlList, e.Request.URL) + case *network.EventResponseReceivedExtraInfo: + // The raw headers, Set-Cookie included. + for k, v := range e.Headers { + value := fmt.Sprint(v) + s.headers = append(s.headers, k+": "+value) + if strings.EqualFold(k, "Set-Cookie") { + for line := range strings.SplitSeq(value, "\n") { + if _, rest, ok := strings.Cut(line, "="); ok { + if v, _, _ := strings.Cut(rest, ";"); len(v) > 16 { + s.cookies = append(s.cookies, v) + } + } + } + } + } + } + }) + return s +} + +func (s *seenTraffic) urls() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.urlList) +} + +func (s *seenTraffic) cookieValues() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.cookies) +} + +// leaks is every URL or response header that holds something shaped like a JWT. +func (s *seenTraffic) leaks() (leaks []string) { + s.mu.Lock() + defer s.mu.Unlock() + for _, text := range append(slices.Clone(s.urlList), s.headers...) { + if m := jwt.FindString(text); m != "" { + leaks = append(leaks, fmt.Sprintf("%.60s: %.24s...", text, m)) + } + } + return leaks +}