diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e800689d7..59d1d4bda 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1864,8 +1864,8 @@ jobs: job_timeout: 75 - bucket: network-isolation parallel: 1 - test_timeout: 15m - job_timeout: 60 + test_timeout: 25m + job_timeout: 75 - bucket: operator parallel: 8 test_timeout: 20m @@ -1951,17 +1951,22 @@ jobs: GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'wake-hostport' && 'two-node' || (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: install real monitoring fixture - if: matrix.bucket == 'monitoring' && env.RUN_REGEX != '' + if: (matrix.bucket == 'monitoring' || matrix.bucket == 'network-isolation') && env.RUN_REGEX != '' run: bash deploy/kind/monitoring.sh - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' env: GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports + E2E_BUCKET: ${{ matrix.bucket }} run: | + prefix=() + if [[ "$E2E_BUCKET" == network-isolation ]]; then + prefix=(bash ../../deploy/kind/with-network-peers.sh) + fi cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ + "${prefix[@]}" go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./... - name: upload measured panel recovery report if: always() && matrix.bucket == 'recovery' diff --git a/changelog/unreleased/894-tenant-network-policies.md b/changelog/unreleased/894-tenant-network-policies.md new file mode 100644 index 000000000..c2cb6863d --- /dev/null +++ b/changelog/unreleased/894-tenant-network-policies.md @@ -0,0 +1,5 @@ +# Tenant network isolation + +The opt-in `networkPolicies.tenantIsolation` profile restricts traffic between game +workloads in managed namespaces. The monitoring guide now distinguishes per-server probe +ingress from the legacy kubelet rule. [PR #894](https://github.com/GameplanePanel/Gameplane/pull/894) diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index ed2ae0a8a..fa2ed06b9 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -1,5 +1,12 @@ {{- if (eq (include "gameplane.operatorEnabled" .) "true") }} {{- if .Values.networkPolicies.enabled }} +{{- $namespaces := list .Values.gamesNamespace }} +{{- if .Values.networkPolicies.tenantIsolation }} +{{- $namespaces = include "gameplane.operatorGameNamespaces" . | fromJsonArray }} +{{- end }} +{{- range $namespace := $namespaces }} +{{- with $ }} +--- # Default-deny ingress in the games namespace. Individual GameServers # open only the ports declared in their template (and whatever the # operator surfaces via the backing Service / Ingress). @@ -7,7 +14,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny-ingress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: {} @@ -23,7 +30,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: {} @@ -33,6 +40,10 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system + {{- if .Values.networkPolicies.tenantIsolation }} + podSelector: + matchLabels: { k8s-app: kube-dns } + {{- end }} ports: - { protocol: UDP, port: 53 } - { protocol: TCP, port: 53 } @@ -47,7 +58,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-agent-to-apiserver - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -78,7 +89,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-api-to-agent - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -110,7 +121,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-api-to-capture - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -150,7 +161,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-prometheus-to-agent - namespace: {{ $.Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" $ | nindent 4 }} spec: podSelector: @@ -168,7 +179,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-prometheus-to-tunnel - namespace: {{ $.Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" $ | nindent 4 }} spec: podSelector: @@ -199,7 +210,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-kubelet-probes - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -237,7 +248,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-game-public-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -279,7 +290,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-backup-restore-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -288,7 +299,9 @@ spec: policyTypes: [Egress] egress: - to: - - ipBlock: { cidr: 0.0.0.0/0 } + {{- range ($backupEgress.toCIDRs | default (list "0.0.0.0/0")) }} + - ipBlock: { cidr: {{ . | quote }} } + {{- end }} ports: {{- range $backupEgress.ports }} - { protocol: {{ .protocol }}, port: {{ .port }} } @@ -306,7 +319,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-ingress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -332,7 +345,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-apiserver - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -356,6 +369,7 @@ spec: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } --- +{{- if not .Values.networkPolicies.tenantIsolation }} # Legacy sentinel proxy allowance, ingress half. This grants ALL ports between # non-isolated sentinels and games. NetworkPolicies are additive; per-server # player ingress does not narrow it. Opted-in templates use operator policies. @@ -369,7 +383,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-game - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -397,7 +411,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-game-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -415,5 +429,8 @@ spec: - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } # Intentionally unrestricted for templates that have not opted in. {{- end }} +{{- end }} +{{- end }} +{{- end }} {{- end }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 9f0d206d5..dce95db8e 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -335,6 +335,9 @@ spec: - --tunnel-frp-image={{ include "gameplane.tunnelFrpImage" . }} - --tunnel-tailscale-image={{ include "gameplane.tunnelTailscaleImage" . }} - --tunnel-playit-image={{ include "gameplane.tunnelPlayitImage" . }} + {{- if .Values.networkPolicies.tenantIsolation }} + - --restrict-tunnel-egress + {{- end }} {{- if .Values.operator.configInitImage }} - --config-init-image={{ .Values.operator.configInitImage }} {{- end }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index e96f3bc26..cad46ba13 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,4 +1,26 @@ {{- $network := .Values.networkPolicies | default dict }} +{{- if $network.tenantIsolation }} +{{- if or (not $network.strict) (not ($network.probeIngress | default dict).enabled) $network.probePorts }} +{{- fail "networkPolicies.tenantIsolation requires strict=true, probeIngress.enabled=true and empty shared probePorts" }} +{{- end }} +{{- if ($network.gameEgress | default dict).enabled }} +{{- fail "networkPolicies.tenantIsolation requires gameEgress.enabled=false; grant approved game downloads with destination-specific policies" }} +{{- end }} +{{- range $cidr := concat ($network.apiServerCIDRs | default list) ($network.kubeletCIDRs | default list) }} +{{- if not (regexMatch "(/32|/128)$" (toString $cidr)) }} +{{- fail "networkPolicies.tenantIsolation requires individual API and kubelet host CIDRs" }} +{{- end }} +{{- end }} +{{- $backup := $network.backupEgress | default (dict "enabled" true) }} +{{- if and $backup.enabled (or (not $backup.ports) (not $backup.toCIDRs)) }} +{{- fail "networkPolicies.tenantIsolation requires explicit backupEgress.toCIDRs and ports, or backupEgress.enabled=false" }} +{{- end }} +{{- range $backup.toCIDRs }} +{{- if hasSuffix "/0" (toString .) }} +{{- fail "networkPolicies.tenantIsolation forbids unrestricted backup destinations" }} +{{- end }} +{{- end }} +{{- end }} {{- if $network.strict }} {{- if not $network.enabled }} {{- fail "networkPolicies.strict requires networkPolicies.enabled=true" }} @@ -17,16 +39,24 @@ {{- $ipv4 := printf "^%s(\\.%s){3}/([0-9]|[12][0-9]|3[0-2])$" $octet $octet }} {{- $h := "[0-9A-Fa-f]{1,4}" }} {{- $ipv6 := printf "^((%s:){7}%s|(%s:){1,7}:|(%s:){1,6}:%s|(%s:){1,5}(:%s){1,2}|(%s:){1,4}(:%s){1,3}|(%s:){1,3}(:%s){1,4}|(%s:){1,2}(:%s){1,5}|%s:((:%s){1,6})|:((:%s){1,7}|:))/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h }} -{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" }} -{{- range (index $network $key) }} +{{- $cidrLists := dict "kubeletCIDRs" $network.kubeletCIDRs "apiServerCIDRs" $network.apiServerCIDRs }} +{{- $portLists := dict "probePorts" $network.probePorts }} +{{- if and $network.tenantIsolation ($network.backupEgress | default dict).enabled }} +{{- $_ := set $cidrLists "backupEgress.toCIDRs" $network.backupEgress.toCIDRs }} +{{- $_ := set $portLists "backupEgress.ports" $network.backupEgress.ports }} +{{- end }} +{{- range $key, $entries := $cidrLists }} +{{- range $entries }} {{- if or (not (kindIs "string" .)) (not (or (regexMatch $ipv4 (toString .)) (regexMatch $ipv6 (toString .)))) }} {{- fail (printf "networkPolicies.%s entries must be explicit IPv4/IPv6 CIDRs" $key) }} {{- end }} {{- end }} {{- end }} -{{- range $network.probePorts }} -{{- if or (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} -{{- fail "networkPolicies.probePorts requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" }} +{{- range $key, $entries := $portLists }} +{{- range $entries }} +{{- if or (not (kindIs "map" .)) (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} +{{- fail (printf "networkPolicies.%s requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" $key) }} +{{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index d20745fd9..f4ea900a9 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -595,6 +595,10 @@ networkPolicies: # Fail closed unless actual kubelet source CIDRs/probe ports and apiserver # endpoint CIDRs are supplied. Defaults retain the legacy broad allowances. strict: false + # Opt in only after provisioning each managed namespace and its provider + # egress. Requires strict endpoints, per-server probes, and explicit backup + # destinations. See the tenant network profile in docs/security.md. + tenantIsolation: false # CIDRs that hold kubelet source IPs for liveness/readiness probes. # If unset, the chart falls back to RFC1918 + link-local ranges, # which cover most clusters without opening the public internet. @@ -648,6 +652,10 @@ networkPolicies: # store) as well as public. backupEgress: enabled: true + # Explicit repository destinations. The compatibility profile defaults + # to any IPv4 address on the configured ports. Tenant isolation requires + # a nonempty list (or backupEgress.enabled=false and administrator policies). + toCIDRs: [] # Ports opened for restic repository connections. Defaults cover # HTTPS-based backends (S3, B2, REST-over-TLS, Swift, Azure, GCS) # and SFTP. Add your repository's port if it uses neither, e.g. a diff --git a/charts/tests/check_network_isolation.py b/charts/tests/check_network_isolation.py index c8c267b7e..3200bf267 100644 --- a/charts/tests/check_network_isolation.py +++ b/charts/tests/check_network_isolation.py @@ -69,6 +69,32 @@ def main(): "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"][0] assert extra["ports"] == [{"protocol": "TCP", "port": 9000}] + tenant = copy.deepcopy(strict) + tenant["operator"] = {"gameNamespaces": ["tenant-a", "tenant-a"]} + tenant["networkPolicies"].update({ + "tenantIsolation": True, + "gameEgress": {"enabled": False}, + "backupEgress": {"enabled": True, "toCIDRs": ["192.0.2.3/32"], + "ports": [{"protocol": "TCP", "port": 443}]}, + }) + policies = [o for o in render(tenant) if o.get("kind") == "NetworkPolicy"] + for ns in ("gameplane-games", "tenant-a"): + local = [p for p in policies if p["metadata"]["namespace"] == ns] + names = [p["metadata"]["name"] for p in local] + assert len(names) == len(set(names)) + assert {"default-deny-ingress", "default-deny-egress", "allow-agent-to-apiserver"} <= set(names) + assert not {"allow-sentinel-to-game", "allow-sentinel-to-game-egress", "allow-kubelet-probes"} & set(names) + backup = one(local, "NetworkPolicy", "allow-backup-restore-egress") + assert backup["spec"]["egress"] == [{"to": [{"ipBlock": {"cidr": "192.0.2.3/32"}}], + "ports": [{"protocol": "TCP", "port": 443}]}] + for field, invalid_value, failure in (("ports", [{}], "integer port"), + ("ports", [{"protocol": "TCP", "port": 0}], "integer port"), + ("toCIDRs", ["192.0.2.999/32"], "must be explicit"), + ("toCIDRs", ["::/0"], "unrestricted backup")): + invalid = copy.deepcopy(tenant) + invalid["networkPolicies"]["backupEgress"][field] = invalid_value + render(invalid, failure=failure) + if __name__ == "__main__": main() diff --git a/deploy/kind/with-network-peers.sh b/deploy/kind/with-network-peers.sh new file mode 100644 index 000000000..32ed7d800 --- /dev/null +++ b/deploy/kind/with-network-peers.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Run the Cilium bucket with reachable, off-cluster endpoints on kind's bridge. +set -euo pipefail +node="$(kind get nodes --name "${CLUSTER:?}" | head -n1)" +[[ -n "$node" && "$(docker inspect --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}' "$node")" == "$CLUSTER" ]] +scratch="$(mktemp -d)" +peer="" repo="" route_added=false +metadata=169.254.169.253 +cleanup() { + if [[ "$route_added" == true ]]; then docker exec "$node" ip route del "$metadata/32" via "$GAMEPLANE_NET_PEER_IP" || true; fi + if [[ -n "$repo" ]]; then docker rm -f "$repo" >/dev/null || true; fi + if [[ -n "$peer" ]]; then docker rm -f "$peer" >/dev/null || true; fi + rm -rf "$scratch" +} +trap cleanup EXIT +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=kind-private-fixture \ + -keyout "$scratch/tls.key" -out "$scratch/tls.crt" >/dev/null 2>&1 +cat >"$scratch/listen.py" <<'PY' +import http.server, ssl, threading + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"isolated-kind-peer\n") + +metadata = http.server.ThreadingHTTPServer(("169.254.169.253", 80), Handler) +threading.Thread(target=metadata.serve_forever, daemon=True).start() +tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +tls.load_cert_chain("/fixture/tls.crt", "/fixture/tls.key") +https = http.server.ThreadingHTTPServer(("0.0.0.0", 443), Handler) +https.socket = tls.wrap_socket(https.socket, server_side=True) +https.serve_forever() +PY +peer="$(docker run --detach --network kind --cap-add NET_ADMIN --volume "$scratch:/fixture:ro" \ + python:3.14-alpine sh -ec 'ip addr add 169.254.169.253/32 dev lo; exec python /fixture/listen.py')" +GAMEPLANE_NET_PEER_IP="$(docker inspect --format '{{(index .NetworkSettings.Networks "kind").IPAddress}}' "$peer")" +[[ -n "$GAMEPLANE_NET_PEER_IP" ]] +# Route only this alternate metadata address inside the disposable node. +# The real cloud metadata address is never used. +docker exec "$node" ip route add "$metadata/32" via "$GAMEPLANE_NET_PEER_IP" +route_added=true +repo="$(docker run --detach --network "container:$peer" --entrypoint rest-server \ + restic/rest-server:0.14.0 --listen :8000 --no-auth --path /tmp/repository)" +GAMEPLANE_NET_NODE_IP="$(docker inspect --format '{{(index .NetworkSettings.Networks "kind").IPAddress}}' "$node")" +export GAMEPLANE_NET_PEER_IP GAMEPLANE_NET_NODE_IP +export GAMEPLANE_NET_PEER_CONTAINER="$peer" +"$@" diff --git a/docs/install.md b/docs/install.md index 68dfff30e..89fd9e581 100644 --- a/docs/install.md +++ b/docs/install.md @@ -207,8 +207,10 @@ operator: ``` These additional namespaces receive operator workload RBAC. Provision their -namespace admission labels, NetworkPolicies, and any API/gateway access settings -separately; those chart resources still target `gamesNamespace`. Registering a +Restricted admission labels and any required API/gateway access before use. +The [tenant network profile](security.md#tenant-network-profile) installs the +chart's baseline NetworkPolicies in every managed namespace. With the compatibility +defaults, provision additional namespaces' NetworkPolicies separately. Registering a remote cluster does not add local namespaces or extend Secret authority: remote health checks use discovery only, while each remote operator has its own scope. diff --git a/docs/monitoring.md b/docs/monitoring.md index 83a22b300..6f6245947 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -39,11 +39,13 @@ on TCP 9090 and `app.kubernetes.io/name: gameplane-tunnel` on TCP 8091, with a namespace selector for the scraper. Restricted Prometheus egress policies must also allow those destinations, DNS and API discovery. -For enforced isolation, configure `networkPolicies.strict: true`, actual -`kubeletCIDRs` and `apiServerCIDRs`, and the game modules' exact `probePorts`. -The legacy broad kubelet allowance can otherwise admit other private-address -sources on metrics ports. Keep metrics ports out of the game probe allowance; -the dedicated scrape policies grant their access. The CI monitoring bucket +For enforced isolation, configure `networkPolicies.strict: true` with the +actual kubelet and API endpoints. With per-server probe ingress enabled, +leave `networkPolicies.probePorts: []`. Each game receives its own probe ports. +Every explicit shared port grants kubelets access to that port on **all** game +pods. If you disable per-server probe ingress, list the shared ports explicitly: +an empty list in that legacy mode opens every game-pod port to the kubelet CIDRs. +Use the dedicated scrape policies for metrics. The CI monitoring bucket uses Cilium and actual node/API addresses, tests a scraper in a separate namespace, revokes/restores the scrape policy, and checks an unapproved namespace cannot reach agent metrics. diff --git a/docs/security.md b/docs/security.md index 35b1db01b..13edd91a4 100644 --- a/docs/security.md +++ b/docs/security.md @@ -306,6 +306,59 @@ intended for local `kind` development where mTLS is overkill. ## NetworkPolicies +### Tenant network profile + +Pre-provision a dedicated namespace per tenant and apply the cluster's restricted +Pod Security Admission labels there. Upgrade each wake-enabled template to +`spec.sentinelIsolation: true`, and verify its advertised ports before switching +profiles. With `networkPolicies.tenantIsolation: true`, Helm owns the baseline +policies in the primary namespace and every namespace in `operator.gameNamespaces`. +Inventory policies with the same names before enabling this on an existing system. +Removing a namespace from the list can remove its Helm-owned policies. Drain +its games or hand policy ownership to the cluster administrator first. + +```yaml +podSecurity: + enforceRestricted: true +operator: + gameNamespaces: [tenant-b] +networkPolicies: + tenantIsolation: true + strict: true + kubeletCIDRs: [192.0.2.10/32, 192.0.2.11/32] + apiServerCIDRs: [192.0.2.2/32] + probeIngress: {enabled: true} + probePorts: [] + gameEgress: {enabled: false} + backupEgress: + enabled: true + toCIDRs: [192.0.2.30/32] + ports: [{protocol: TCP, port: 443}] +``` + +Replace the example addresses with endpoints observed by the enforcing CNI +after Service translation. The profile omits legacy cross-game sentinel rules, +limits DNS to kube-dns-labeled pods in kube-system, and admits both tunnel egress +and backend ingress between a relay and its own game/waker on advertised ports. Provide scoped DNS allowances if +the cluster uses NodeLocal DNS. Open approved asset downloads explicitly. +Use a namespace-local policy for each relay's provider control/data endpoints. +Select its `app.kubernetes.io/name: gameplane-tunnel` and +`app.kubernetes.io/instance: ` labels. For Playit also allow the exact +Kubernetes API endpoint for address publication. Providers with dynamic +destinations require a provider-aware egress policy or a separately isolated +deployment. Check existing provider policies after the operator reconciles. + +Run positive joins, heartbeat, scrape and backup checks on the deployed CNI. +When upgrading with Helm's `--reuse-values` from a shared kubelet probe list, +pass `--set-json 'networkPolicies.probePorts=[]'` to replace the previous +list explicitly. Check that each managed game namespace has its per-server +probe policies and that the legacy `allow-kubelet-probes` grant is absent. +From an independent tenant, check the target Pod IP and Service for RCON and +agent access. Verify private HTTPS and metadata endpoints are unreachable. +Review additional policies in each namespace because their grants are additive. + +### Chart and operator policy rules + When `networkPolicies.enabled=true` (default) the chart applies: - `default-deny-ingress` — denies all ingress to all pods in the games @@ -359,9 +412,9 @@ When `networkPolicies.enabled=true` (default) the chart applies: - `allow-prometheus-to-agent` — opt-in (rendered only when `serviceMonitors.scrapeNamespaceSelector` is set) policy allowing Prometheus pods in the selected namespace to reach the agent's plain-HTTP metrics port - (TCP 9090). That port is unauthenticated HTTP, not mTLS, and is otherwise - already reachable from `kubeletCIDRs` via `allow-kubelet-probes` above - unless `networkPolicies.probePorts` is narrowed to exclude it. + (TCP 9090). The per-server probe policy opens it only for a game that uses + that probe port. A shared `probePorts` entry for 9090 or the legacy all-port + kubelet allowance can also grant access to this unauthenticated listener. - `allow-game-public-egress` — (enabled by default, gated by `networkPolicies.gameEgress.enabled`) allows game pods to reach the public internet for binary/asset/mod downloads. Set `networkPolicies.gameEgress.enabled: false` diff --git a/operator/cmd/main.go b/operator/cmd/main.go index da49633fe..dc2e65817 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -131,6 +131,7 @@ func main() { tunnelFrpImage string tunnelTailscaleImage string tunnelPlayitImage string + restrictTunnelEgress bool agentLogLevel string agentCABundle string agentClientCert string @@ -217,6 +218,8 @@ func main() { flag.StringVar(&tunnelTailscaleImage, "tunnel-tailscale-image", controller.DefaultTunnelTailscaleImage, "Image for the Tailscale tunnel relay pod that routes players through a Tailscale tailnet. "+ "Point at a private registry mirror for air-gapped installs.") + flag.BoolVar(&restrictTunnelEgress, "restrict-tunnel-egress", false, + "Only grant tunnel egress to this server's advertised backend ports; administrators supply provider egress policies.") flag.StringVar(&tunnelPlayitImage, "tunnel-playit-image", controller.DefaultTunnelPlayitImage, "Image for the Playit tunnel relay pod that routes players through a Playit tunnel. "+ "Point at a private registry mirror for air-gapped installs.") @@ -515,6 +518,7 @@ func main() { TunnelFrpImage: tunnelFrpImage, TunnelTailscaleImage: tunnelTailscaleImage, TunnelPlayitImage: tunnelPlayitImage, + RestrictTunnelEgress: restrictTunnelEgress, AgentLogLevel: agentLogLevel, AgentCASecretName: agentCASecretName, AgentCASecretNamespace: agentCASecretNamespace, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index a873b9354..f05741114 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -103,6 +103,7 @@ type GameServerReconciler struct { TunnelFrpImage string TunnelTailscaleImage string TunnelPlayitImage string + RestrictTunnelEgress bool // AgentCASecretName / AgentCASecretNamespace point at the cluster- // wide Secret holding `ca.crt` + `ca.key` used to sign the @@ -451,14 +452,14 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // it will advertise. Like planSentinel, this is computed once per pass so // there is a single source of truth (see planTunnel's doc comment). tunnelPlan := r.planTunnel(ctx, &gs, &tmpl) - if err := r.reconcileTunnel(ctx, &gs, &tmpl, tunnelPlan.wantTunnel); err != nil { - logger.Error(err, "reconcile tunnel") - return ctrl.Result{}, err - } if err := r.reconcileTunnelNetworkPolicy(ctx, &gs, &tmpl, tunnelPlan); err != nil { logger.Error(err, "reconcile tunnel NetworkPolicy") return ctrl.Result{}, err } + if err := r.reconcileTunnel(ctx, &gs, &tmpl, tunnelPlan.wantTunnel); err != nil { + logger.Error(err, "reconcile tunnel") + return ctrl.Result{}, err + } if err := r.reconcileService(ctx, &gs, &tmpl, plan.routeToSentinel); err != nil { logger.Error(err, "reconcile Service") return ctrl.Result{}, err diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 5c46abfb9..ae9875706 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "regexp" "strconv" @@ -551,145 +552,128 @@ func (r *GameServerReconciler) clearTunnelCredentialRefused(ctx context.Context, return nil } -// reconcileTunnelNetworkPolicy maintains a per-server NetworkPolicy admitting -// the tunnel pod's outbound traffic. The games namespace runs a default-deny- -// egress policy (allow-game-public-egress selects app.kubernetes.io/name=gameplane-game), -// which would silently drop the tunnel pod's connection to the relay without -// an explicit admit rule. Without this, the tunnel feature silently does nothing. -// -// The policy admits egress to any destination on the advertised ports and DNS. +// The restricted profile admits both halves of the same-namespace relay path. +// Legacy provider/DNS egress remains available when the profile is disabled. func (r *GameServerReconciler) reconcileTunnelNetworkPolicy( ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, plan tunnelPlan, ) error { - np := &networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: gs.Name + "-tunnel-egress", - Namespace: gs.Namespace, - }, - } + tunnel := metav1.LabelSelector{MatchLabels: map[string]string{ + tunnelLabel: tunnelValue, "app.kubernetes.io/instance": gs.Name, + }} + backend := metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/instance": gs.Name}, + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: tunnelLabel, Operator: metav1.LabelSelectorOpIn, Values: []string{"gameplane-game", "gameplane-waker"}, + }}, + } + var egress, ingress *networkingv1.NetworkPolicySpec + if plan.wantTunnel { + egress = &networkingv1.NetworkPolicySpec{PodSelector: tunnel, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}} + ports := networkPolicyPortsFromTemplate(tmpl) + if !r.RestrictTunnelEgress { + egress.Egress = legacyTunnelEgress(gs.Spec.Networking.Tunnel, ports) + } else if len(ports) > 0 { + egress.Egress = []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{PodSelector: &backend}}, Ports: ports, + }} + ingress = &networkingv1.NetworkPolicySpec{ + PodSelector: backend, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &tunnel}}, Ports: ports, + }}, + } + } + } + return errors.Join( + r.applyTunnelPolicy(ctx, gs, "-tunnel-egress", egress), + r.applyTunnelPolicy(ctx, gs, "-tunnel-ingress", ingress), + ) +} - if !plan.wantTunnel { - // Converge-to-absent: only delete if we own it. - var existing networkingv1.NetworkPolicy - if err := r.Get(ctx, client.ObjectKeyFromObject(np), &existing); err != nil { - return client.IgnoreNotFound(err) - } - if !metav1.IsControlledBy(&existing, gs) { - return nil - } - return client.IgnoreNotFound(r.Delete(ctx, &existing)) +func (r *GameServerReconciler) applyTunnelPolicy(ctx context.Context, gs *gameplanev1alpha1.GameServer, suffix string, spec *networkingv1.NetworkPolicySpec) error { + np := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + suffix, Namespace: gs.Namespace}} + if spec == nil { + return r.deleteIfControlledBy(ctx, gs, np) } - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { - // Select the tunnel pod's labels. - np.Spec.PodSelector = metav1.LabelSelector{ - MatchLabels: map[string]string{ - tunnelLabel: tunnelValue, - "app.kubernetes.io/instance": gs.Name, - }, + if np.ResourceVersion != "" && !metav1.IsControlledBy(np, gs) { + return fmt.Errorf("tunnel policy %s/%s is not owned by this GameServer", np.Namespace, np.Name) } + np.Spec = *spec + return controllerutil.SetControllerReference(gs, np, r.Scheme) + }) + return err +} - // Egress rules: allow DNS, advertised ports for inbound relay traffic, - // and the provider's relay endpoint ports for outbound control traffic. - egressPorts := make([]networkingv1.NetworkPolicyPort, 0) - - // DNS: UDP and TCP port 53 to any destination (required by all providers). - tcpProto := corev1.ProtocolTCP - udpProto := corev1.ProtocolUDP - dnsPort := intstr.FromInt(53) - egressPorts = append(egressPorts, - networkingv1.NetworkPolicyPort{ - Protocol: &udpProto, - Port: &dnsPort, - }, - networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &dnsPort, - }, - ) - - // Provider relay egress: the ports the tunnel process dials outbound. - tunnel := gs.Spec.Networking.Tunnel - if tunnel != nil { - switch tunnel.Provider { - case "frp": - // frp: outbound to the frp server on ServerAddr:ServerPort (default 7000). - if tunnel.Frp != nil { - serverPort := tunnel.Frp.ServerPort - if serverPort == 0 { - serverPort = 7000 - } - port := intstr.FromInt32(serverPort) - egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &port, - }) - } - - case "tailscale": - // tailscale: outbound to control plane on TCP 443 and DERP relays on UDP 41641. - httpsPort := intstr.FromInt(443) - derpPort := intstr.FromInt(41641) - egressPorts = append(egressPorts, - networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &httpsPort, - }, - networkingv1.NetworkPolicyPort{ - Protocol: &udpProto, - Port: &derpPort, - }, - ) - - case "playit": - // playit does not publish a fixed set of relay endpoints or ports: - // the playit agent dials its control plane and relay nodes on - // varying TCP and UDP ports that are not known ahead of time. - // A per-port allow list (like frp's ServerPort or tailscale's - // 443/41641) can't express that, so playit gets its own egress - // rule below with no Ports field at all -- which, per the K8s - // NetworkPolicy semantics, means all ports/protocols -- to any - // destination, matching how the frp/tailscale rule above omits - // "To" to allow any destination. The DNS and advertised-port - // rule built here is left untouched for playit. - } - } +func legacyTunnelEgress(tunnel *gameplanev1alpha1.GameServerTunnel, advertised []networkingv1.NetworkPolicyPort) []networkingv1.NetworkPolicyEgressRule { + // Egress rules: allow DNS, advertised ports for inbound relay traffic, + // and the provider's relay endpoint ports for outbound control traffic. + egressPorts := make([]networkingv1.NetworkPolicyPort, 0) + + // DNS: UDP and TCP port 53 to any destination (required by all providers). + tcpProto := corev1.ProtocolTCP + udpProto := corev1.ProtocolUDP + dnsPort := intstr.FromInt(53) + egressPorts = append(egressPorts, + networkingv1.NetworkPolicyPort{ + Protocol: &udpProto, + Port: &dnsPort, + }, + networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &dnsPort, + }, + ) - // Advertised ports: the game's container ports (tunnel receives here, forwards inbound). - for _, p := range tmpl.Spec.Ports { - if !p.Advertise { - continue - } - proto := p.Protocol - if proto == "" { - proto = corev1.ProtocolTCP + // Provider relay egress: the ports the tunnel process dials outbound. + if tunnel != nil { + switch tunnel.Provider { + case "frp": + // frp: outbound to the frp server on ServerAddr:ServerPort (default 7000). + if tunnel.Frp != nil { + serverPort := tunnel.Frp.ServerPort + if serverPort == 0 { + serverPort = 7000 + } + port := intstr.FromInt32(serverPort) + egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &port, + }) } - port := intstr.FromInt32(p.ContainerPort) - egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ - Protocol: &proto, - Port: &port, - }) - } - - // Egress with no "To" means "to any destination". - np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} - egressRules := []networkingv1.NetworkPolicyEgressRule{{ - Ports: egressPorts, - // Empty To = allow to any destination - }} - if tunnel != nil && tunnel.Provider == "playit" { - // playit's relay/control-plane ports aren't statically known, so - // grant it a separate rule with no Ports (= all ports/protocols) - // and no To (= any destination), on top of the DNS/advertised-port - // rule above. - egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{}) - } - np.Spec.Egress = egressRules - return controllerutil.SetControllerReference(gs, np, r.Scheme) - }) - return err + case "tailscale": + // tailscale: outbound to control plane on TCP 443 and DERP relays on UDP 41641. + httpsPort := intstr.FromInt(443) + derpPort := intstr.FromInt(41641) + egressPorts = append(egressPorts, + networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &httpsPort, + }, + networkingv1.NetworkPolicyPort{ + Protocol: &udpProto, + Port: &derpPort, + }, + ) + + } + } + + // Egress with no "To" means "to any destination". + egressRules := []networkingv1.NetworkPolicyEgressRule{{ + Ports: append(egressPorts, advertised...), + // Empty To = allow to any destination + }} + if tunnel != nil && tunnel.Provider == "playit" { + // playit's relay/control-plane ports aren't statically known, so + // grant it a separate rule with no Ports (= all ports/protocols) + // and no To (= any destination), on top of the DNS/advertised-port + // rule above. + egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{}) + } + return egressRules } // getTunnel fetches the tunnel Deployment, reporting exists=false when absent. diff --git a/operator/internal/controller/gameserver_tunnel_test.go b/operator/internal/controller/gameserver_tunnel_test.go index 2b23a570e..184969060 100644 --- a/operator/internal/controller/gameserver_tunnel_test.go +++ b/operator/internal/controller/gameserver_tunnel_test.go @@ -1,13 +1,121 @@ package controller import ( + "reflect" "testing" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) +func TestRestrictTunnelEgressReplacesLegacyPlayitRules(t *testing.T) { + scheme := testScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + r := &GameServerReconciler{Scheme: scheme, Client: fake.NewClientBuilder().WithScheme(scheme).Build()} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "one"}} + gs.Spec.Networking.Tunnel = &gameplanev1alpha1.GameServerTunnel{Provider: "playit"} + tmpl := singlePortTemplate(true, "minecraft") + reconcile := func() []networkingv1.NetworkPolicyEgressRule { + t.Helper() + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{wantTunnel: true}); err != nil { + t.Fatal(err) + } + var np networkingv1.NetworkPolicy + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}, &np); err != nil { + t.Fatal(err) + } + return np.Spec.Egress + } + if rules := reconcile(); len(rules) != 2 { + t.Fatalf("unexpected compatibility rules: %+v", rules) + } + r.RestrictTunnelEgress = true + rules := reconcile() + if len(rules) != 1 || len(rules[0].To) != 1 || !reflect.DeepEqual(rules[0].Ports, networkPolicyPortsFromTemplate(tmpl)) { + t.Fatalf("unexpected restricted rules: %+v", rules) + } + peer := rules[0].To[0] + if peer.PodSelector == nil || peer.NamespaceSelector != nil || peer.IPBlock != nil { + t.Fatalf("expected a pod peer in the local namespace: %+v", peer) + } + sel, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct { + name, instance string + allowed bool + }{ + {"gameplane-game", "alpha", true}, {"gameplane-waker", "alpha", true}, + {"gameplane-game", "beta", false}, {"gameplane-operator", "alpha", false}, + } { + if sel.Matches(labels.Set{"app.kubernetes.io/name": target.name, "app.kubernetes.io/instance": target.instance}) != target.allowed { + t.Fatalf("unexpected peer match for %+v", target) + } + } + var ingress networkingv1.NetworkPolicy + inKey := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-ingress"} + if err := r.Get(t.Context(), inKey, &ingress); err != nil { + t.Fatal(err) + } + wantTunnel := metav1.LabelSelector{MatchLabels: map[string]string{ + tunnelLabel: tunnelValue, "app.kubernetes.io/instance": gs.Name, + }} + wantRules := []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &wantTunnel}}, Ports: rules[0].Ports, + }} + if !reflect.DeepEqual(ingress.Spec.PodSelector, *peer.PodSelector) || + !reflect.DeepEqual(ingress.Spec.Ingress, wantRules) || !metav1.IsControlledBy(&ingress, gs) { + t.Fatalf("unexpected ingress policy: %+v", ingress) + } + absent := func(key client.ObjectKey) { + t.Helper() + if err := r.Get(t.Context(), key, &networkingv1.NetworkPolicy{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected %s to be absent, got %v", key, err) + } + } + r.RestrictTunnelEgress = false + reconcile() + absent(inKey) + r.RestrictTunnelEgress = true + reconcile() + tmpl.Spec.Ports = nil + if rules := reconcile(); len(rules) != 0 { + t.Fatalf("empty advertised port set grants egress: %+v", rules) + } + absent(inKey) + tmpl = singlePortTemplate(true, "minecraft") + reconcile() + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{}); err != nil { + t.Fatal(err) + } + absent(inKey) + absent(client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}) + + foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: inKey.Name, Namespace: inKey.Namespace, UID: "foreign"}} + if err := r.Create(t.Context(), foreign); err != nil { + t.Fatal(err) + } + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{wantTunnel: true}); err == nil { + t.Fatal("accepted foreign ingress policy") + } + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{}); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), inKey, &ingress); err != nil || ingress.UID != foreign.UID || len(ingress.OwnerReferences) != 0 { + t.Fatalf("changed foreign ingress policy: %+v, %v", ingress, err) + } +} + // TestBuildFrpRemotePortsConfig covers F-052: the BACKING_SERVICE_PORT env // var must carry the template port's own containerPort and protocol // alongside the operator-chosen public remotePort, not just the remotePort diff --git a/operator/internal/controller/owned_delete.go b/operator/internal/controller/owned_delete.go index 69845e5f7..84e9e340e 100644 --- a/operator/internal/controller/owned_delete.go +++ b/operator/internal/controller/owned_delete.go @@ -38,6 +38,7 @@ func (r *GameServerReconciler) deleteIfControlledBy( // the ownership check above and this Delete reaching the API server // would still match on name and be deleted, even though this GameServer // never controlled the replacement. - uid := obj.GetUID() - return client.IgnoreNotFound(r.Delete(ctx, obj, client.Preconditions{UID: &uid})) + // Also fence ownership changes to the same object between the read and delete. + uid, version := obj.GetUID(), obj.GetResourceVersion() + return client.IgnoreNotFound(r.Delete(ctx, obj, client.Preconditions{UID: &uid, ResourceVersion: &version})) } diff --git a/operator/internal/controller/owned_delete_test.go b/operator/internal/controller/owned_delete_test.go index 3dd8907d4..363460b10 100644 --- a/operator/internal/controller/owned_delete_test.go +++ b/operator/internal/controller/owned_delete_test.go @@ -190,6 +190,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { controllerutil.SetControllerReference(gs, secret, scheme) var gotUID *types.UID + var gotVersion *string cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gs, secret). WithInterceptorFuncs(interceptor.Funcs{ Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { @@ -197,6 +198,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { do.ApplyOptions(opts) if do.Preconditions != nil { gotUID = do.Preconditions.UID + gotVersion = do.Preconditions.ResourceVersion } return c.Delete(ctx, obj, opts...) }, @@ -213,4 +215,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { if *gotUID != secret.UID { t.Errorf("Delete UID precondition = %q, want %q", *gotUID, secret.UID) } + if gotVersion == nil || *gotVersion == "" || *gotVersion != secret.ResourceVersion { + t.Fatalf("expected delete fenced at observed resourceVersion %q, got %v", secret.ResourceVersion, gotVersion) + } } diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go index 20a1057a3..f76573b7e 100644 --- a/operator/internal/controller/sentinel_isolation.go +++ b/operator/internal/controller/sentinel_isolation.go @@ -133,6 +133,8 @@ func (r *GameServerReconciler) revokeInvalidSentinelIsolationExposure(ctx contex &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gameIngressPolicyName(gs)}}, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-ingress"}}, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-egress"}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-ingress"}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-game-direct"}}, } diff --git a/operator/internal/controller/sentinel_isolation_review_test.go b/operator/internal/controller/sentinel_isolation_review_test.go index d3859e210..b9e3caf8d 100644 --- a/operator/internal/controller/sentinel_isolation_review_test.go +++ b/operator/internal/controller/sentinel_isolation_review_test.go @@ -120,7 +120,7 @@ func TestInvalidIsolatedTemplateRevokesOnlyOwnedExposure(t *testing.T) { owner.UID = "foreign-server" } var exposures []client.Object - for _, name := range []string{gameIngressPolicyName(gs), gs.Name + "-sentinel-ingress", gs.Name + "-sentinel-egress"} { + for _, name := range []string{gameIngressPolicyName(gs), gs.Name + "-sentinel-ingress", gs.Name + "-sentinel-egress", gs.Name + "-tunnel-ingress", gs.Name + "-tunnel-egress"} { exposures = append(exposures, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name), OwnerReferences: []metav1.OwnerReference{owner}}}) } for _, name := range []string{gs.Name, gs.Name + "-game-direct"} { diff --git a/operator/specs.md b/operator/specs.md index e54d516fe..d445a71cf 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -373,6 +373,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. + - Tunnel NetworkPolicies use the relay's provider ports in compatibility mode. With `--restrict-tunnel-egress`, a server's tunnel gets egress to its own game or waker and the backend gets the matching ingress from that tunnel. Both rules select only the same namespace, server instance and advertised port/protocol. The chart covers DNS and administrators supply provider egress. - Probe NetworkPolicy (`probe_policy.go`): with `--probe-ingress-policy`, reconcile an owned `-probes` policy admitting `--probe-ingress-from-cidr` to exactly the ports the game container's tcpSocket, httpGet (no `host`) and grpc liveness, readiness and startup probes use; named ports stay named. Exec probes need no network. With the flag off, no network probe, or no CIDRs, delete the policy if the GameServer controls it (a rule with no ports would admit every port). Replaces the chart's catch-all `allow-kubelet-probes` rule. - Template env ownership: Secret and ConfigMap references in the GameTemplate's and GameVersion's `env` must pass the same ownership rule as `spec.env` (`validateTemplateEnvSources` reuses `validateServerEnvSources`), so a template cannot mount another server's Secret into this server's pod. - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. An opted-in template with zero advertised ports revokes its owned game-ingress and sentinel policies plus public/game-direct Services through uncached, UID/resourceVersion-guarded cleanup before reporting actionable `Ready=False`; it creates no workloads or unrestricted grants, preserves foreign resources and the agent Service, and reports the failure even when cleanup needs a retry. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. On opt-out, roll the labels off both workloads and retain the per-server grants until uncached reads confirm both controllers observed the unlabeled templates and completed their rollouts, and neither game nor sentinel pods still carry the label. Settled opt-outs first probe cached sentinel policies and skip live rollout reads when no owned grants exist. The owned NetworkPolicy watch must remain enabled: delayed cache visibility enqueues another cleanup pass. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 1b12f3b4a..12c05206e 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -11,9 +11,12 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" ) // This bucket uses Cilium, not kindnet. Exercise the effective union of all @@ -33,18 +36,18 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { for _, pair := range [][2]string{{a, legacyTemplate}, {b, legacyTemplate}, {c, isolatedTemplate}} { applyBusyboxGameServer(t, ns, pair[0], pair[1]) } - aIP := isolationGameIP(t, a, false) - bIP := isolationGameIP(t, b, false) - cIP := isolationGameIP(t, c, true) - probeA := isolationProbe(t, a, false) - probeC := isolationProbe(t, c, true) + aIP := isolationGameIP(t, ns, a, false) + bIP := isolationGameIP(t, ns, b, false) + cIP := isolationGameIP(t, ns, c, true) + probeA := isolationProbe(t, ns, a, false) + probeC := isolationProbe(t, ns, c, true) // Compatibility: absent-field templates keep their existing broad grant. - isolationHTTP(t, probeA, bIP, 25575, true) - isolationHTTP(t, probeC, cIP, 25566, true) - isolationHTTP(t, probeC, cIP, 25575, false) - isolationHTTP(t, probeC, aIP, 25566, false) - isolationHTTP(t, probeA, cIP, 25566, false) + isolationHTTP(t, ns, probeA, bIP, 25575, true) + isolationHTTP(t, ns, probeC, cIP, 25566, true) + isolationHTTP(t, ns, probeC, cIP, 25575, false) + isolationHTTP(t, ns, probeC, aIP, 25566, false) + isolationHTTP(t, ns, probeA, cIP, 25566, false) ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) if err != nil { @@ -56,7 +59,7 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { if err != nil { t.Fatal(err) } - aIP, bIP = isolationGameIP(t, a, true), isolationGameIP(t, b, true) + aIP, bIP = isolationGameIP(t, ns, a, true), isolationGameIP(t, ns, b, true) ss, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) if err != nil || !reflect.DeepEqual(oldSelector, ss.Spec.Selector) { t.Fatalf("opt-in must preserve the immutable StatefulSet selector: %v", err) @@ -66,10 +69,10 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { if err != nil { t.Fatal(err) } - isolationHTTP(t, probeA, aIP, 25566, true) - isolationHTTP(t, probeA, aIP, 25575, false) - isolationHTTP(t, probeA, bIP, 25566, false) - isolationHTTP(t, probeA, bIP, 25575, false) + isolationHTTP(t, ns, probeA, aIP, 25566, true) + isolationHTTP(t, ns, probeA, aIP, 25575, false) + isolationHTTP(t, ns, probeA, bIP, 25566, false) + isolationHTTP(t, ns, probeA, bIP, 25575, false) // Kubelet HTTP readiness and agent heartbeats still work with strict CIDRs. // Also exercise an actual API -> agent mTLS call through its Service. @@ -87,13 +90,179 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { stamp, err := time.Parse(time.RFC3339, hb) return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" }) + + checkTenantNetworkProfile(t, ns, a, b, probeA) +} + +func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { + t.Helper() + ctx := t.Context() + stamp := time.Now().UnixNano() + // Exercise the opt-in upgrade after checking the historical compatibility + // path. Identically named servers in separate namespaces must not become + // each other's relay backend. + tenant := fmt.Sprintf("tenant-net-%d", stamp) + _, err := envInstance.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: tenant, Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "restricted", "pod-security.kubernetes.io/enforce-version": "v1.34", + }}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) + }) + peerIP := kindFixtureIP(t, "GAMEPLANE_NET_PEER_IP") + helmUpgradeReuseJSON(ctx, t, []string{"networkPolicies.probePorts=[]"}, + "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", + "podSecurity.enforceRestricted=true", "networkPolicies.gameEgress.enabled=false", + "networkPolicies.backupEgress.enabled=true", "networkPolicies.backupEgress.toCIDRs[0]="+peerIP+"/32", + "networkPolicies.backupEgress.ports[0].port=8000", "networkPolicies.backupEgress.ports[0].protocol=TCP", + "networkPolicies.gameIngress.fromCIDRs[0]="+peerIP+"/32") + profileApplied := time.Now().UTC().Truncate(time.Second) + secondTemplate := fmt.Sprintf("second-probe-%d", stamp) + isolationTemplate(t, secondTemplate, true) + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, secondTemplate, types.MergePatchType, + []byte(`{"spec":{"probes":{"startup":{"tcpSocket":{"port":"management"}},"readiness":{"httpGet":{"port":25575}}}}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, tenant, a, secondTemplate) + otherIP := isolationGameIP(t, tenant, a, true) + aIP := isolationGameIP(t, ns, a, true) + otherProbe := isolationProbe(t, tenant, a, true) + isolationHTTP(t, tenant, otherProbe, otherIP, 25566, true) + isolationHTTP(t, ns, probeA, otherIP, 25575, false) + isolationHTTP(t, tenant, otherProbe, aIP, 25575, false) + // Both Services resolve and are reachable from their own sentinels. + // Cross-tenant denials through the same DNS names exercise Service NAT. + home := a + "." + ns + ".svc" + away := a + "." + tenant + ".svc" + isolationHTTP(t, ns, probeA, home, 25566, true) + isolationHTTP(t, tenant, otherProbe, away, 25566, true) + isolationHTTP(t, ns, probeA, away, 25566, false) + isolationHTTP(t, tenant, otherProbe, home, 25566, false) + for namespace, want := range map[string]int32{ns: 25566, tenant: 25575} { + policies := envInstance.K8s.NetworkingV1().NetworkPolicies(namespace) + if _, err := policies.Get(ctx, "allow-kubelet-probes", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("shared kubelet grant survived tenant migration in %s: %v", namespace, err) + } + policy, err := policies.Get(ctx, a+"-probes", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if len(policy.Spec.Ingress) != 1 || len(policy.Spec.Ingress[0].Ports) == 0 { + t.Fatalf("missing restricted kubelet rule for %s/%s: %+v", namespace, a, policy.Spec) + } + rule := policy.Spec.Ingress[0] + if len(rule.From) != 1 || rule.From[0].IPBlock == nil || rule.From[0].IPBlock.CIDR != kindFixtureIP(t, "GAMEPLANE_NET_NODE_IP")+"/32" { + t.Fatalf("unexpected kubelet sources for %s/%s: %+v", namespace, a, rule.From) + } + pod, err := envInstance.K8s.CoreV1().Pods(namespace).Get(ctx, a+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + // Startup names the socket while readiness uses its number. Kubernetes + // resolves a named NetworkPolicy port against the destination Pod. + namedTCP := map[string]int32{} + for _, c := range pod.Spec.Containers { + for _, port := range c.Ports { + if port.Protocol == corev1.ProtocolTCP && port.Name != "" { + namedTCP[port.Name] = port.ContainerPort + } + } + } + for _, port := range rule.Ports { + if port.Port == nil || port.EndPort != nil || port.Protocol == nil || *port.Protocol != corev1.ProtocolTCP { + t.Fatalf("unexpected kubelet grant for %s/%s: %+v", namespace, a, port) + } + effective := port.Port.IntVal + if port.Port.Type == intstr.String { + effective = namedTCP[port.Port.StrVal] + } + if effective != want { + t.Fatalf("kubelet port %v resolves to %d in %s/%s, want only TCP %d", port.Port, effective, namespace, a, want) + } + } + } + + checkTenantEndpoints(t, ns, tenant, a, b, aIP, otherIP, peerIP, profileApplied) + checkTunnelPolicy(t, ns, a, b, probeA, aIP, home, away) + checkSharedProbeFallback(t, ns, a, secondTemplate) +} + +func checkTunnelPolicy(t *testing.T, ns, a, b, probeA, aIP, home, away string) { + t.Helper() + ctx := t.Context() + enableIsolatedFRP(t, ns, a) + tunnel := isolationClient(t, ns, a+"-relay-probe", map[string]string{ + "app.kubernetes.io/name": "gameplane-tunnel", "app.kubernetes.io/instance": a, + }) + isolationHTTP(t, ns, tunnel, aIP, 25566, true) + isolationHTTP(t, ns, tunnel, home, 25566, true) + isolationHTTP(t, ns, tunnel, aIP, 25575, false) + isolationHTTP(t, ns, tunnel, b+"."+ns+".svc", 25566, false) + isolationHTTP(t, ns, tunnel, away, 25566, false) + // Stand up a listener on the existing synthetic waker. The game is running, + // so its main service continues to route exclusively to its game pod. + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, probeA, "--", "sh", "-c", + "mkdir -p /tmp/wake && echo awake >/tmp/wake/index.html && httpd -p 25566 -h /tmp/wake"); err != nil { + t.Fatalf("start wake listener: %v: %s", err, out) + } + waker, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, probeA, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + isolationHTTP(t, ns, tunnel, waker.Status.PodIP, 25566, true) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, a, types.MergePatchType, + []byte(`{"spec":{"networking":{"tunnel":{"enabled":false}}}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + isolationHTTP(t, ns, tunnel, aIP, 25566, false) +} + +func checkSharedProbeFallback(t *testing.T, ns, a, secondTemplate string) { + t.Helper() + ctx := t.Context() + // Exercise the explicit shared-probe fallback in the primary namespace. + // Both servers have active readiness probes but need different ports. + fallbackGame := fmt.Sprintf("net-fallback-%d", time.Now().UnixNano()) + applyBusyboxGameServer(t, ns, fallbackGame, secondTemplate) + isolationGameIP(t, ns, fallbackGame, true) + helmUpgradeReuse(ctx, t, "networkPolicies.tenantIsolation=false", "networkPolicies.probeIngress.enabled=false", + "networkPolicies.probePorts[0].port=25566", "networkPolicies.probePorts[0].protocol=TCP", + "networkPolicies.probePorts[1].port=25575", "networkPolicies.probePorts[1].protocol=TCP") + policies := envInstance.K8s.NetworkingV1().NetworkPolicies(ns) + shared, err := policies.Get(ctx, "allow-kubelet-probes", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(shared.Spec.PodSelector.MatchLabels, map[string]string{"app.kubernetes.io/name": "gameplane-game"}) || + len(shared.Spec.Ingress) != 1 || len(shared.Spec.Ingress[0].Ports) != 2 { + t.Fatalf("unexpected shared kubelet rule: %+v", shared.Spec) + } + for i, want := range []int{25566, 25575} { + port := shared.Spec.Ingress[0].Ports[i] + if port.Protocol == nil || *port.Protocol != corev1.ProtocolTCP || port.Port == nil || port.Port.IntValue() != want { + t.Fatalf("unexpected shared probe port: %+v", port) + } + } + for _, server := range []string{a, fallbackGame} { + envInstance.Eventually(t, time.Minute, func() (bool, string) { + _, err := policies.Get(ctx, server+"-probes", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("per-game probe rule for %s still present: %v", server, err) + }) + isolationGameIP(t, ns, server, true) + } } func isolationTemplate(t *testing.T, name string, isolated bool) { t.Helper() spec := map[string]any{ "displayName": name, "game": "busybox", "version": "1", "image": "busybox:1.36", - "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, + "security": map[string]any{"runAsUser": int64(1000), "runAsGroup": int64(1000), "fsGroup": int64(1000)}, + "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, "ports": []any{ map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}, map[string]any{"name": "management", "containerPort": int64(25575), "protocol": "TCP", "advertise": false}, @@ -118,17 +287,25 @@ func isolationTemplate(t *testing.T, name string, isolated bool) { }) } -func isolationGameIP(t *testing.T, name string, isolated bool) string { +func isolationGameIP(t *testing.T, ns, name string, isolated bool) string { t.Helper() + namespace, err := envInstance.K8s.CoreV1().Namespaces().Get(t.Context(), ns, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + restricted := namespace.Labels["pod-security.kubernetes.io/enforce"] == "restricted" var ip string envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { - pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name+"-0", metav1.GetOptions{}) if err != nil { return false, err.Error() } if pod.DeletionTimestamp != nil || (pod.Labels["gameplane.local/sentinel-isolation"] == "true") != isolated { return false, "waiting for rollout with expected isolation label" } + if restricted && pod.Annotations["gameplane.local/workload-security"] != "restricted" { + return false, "waiting for replacement game pod after the restricted profile rollout" + } for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { ip = pod.Status.PodIP @@ -140,20 +317,31 @@ func isolationGameIP(t *testing.T, name string, isolated bool) string { return ip } -// A synthetic client carries the exact real sentinel selectors. It exposes no -// listener and owns no Service/Deployment, so the running game is unaffected. -func isolationProbe(t *testing.T, server string, isolated bool) string { +// A synthetic client carries the real sentinel selectors in its game namespace. +func isolationProbe(t *testing.T, ns, server string, isolated bool) string { t.Helper() - const ns = "gameplane-games" - name := server + "-probe" labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} if isolated { labels["gameplane.local/sentinel-isolation"] = "true" } + return isolationClient(t, ns, server+"-probe", labels) +} + +func isolationClient(t *testing.T, ns, name string, labels map[string]string) string { + t.Helper() noToken := false pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, - Containers: []corev1.Container{{Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}}}}, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptr.To(true), RunAsUser: ptr.To(int64(1000)), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Containers: []corev1.Container{{ + Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, } if _, err := envInstance.K8s.CoreV1().Pods(ns).Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { t.Fatal(err) @@ -171,16 +359,21 @@ func isolationProbe(t *testing.T, server string, isolated bool) string { return name } -func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { +func isolationHTTP(t *testing.T, ns, source, ip string, port int, allowed bool) { t.Helper() // Every exec must succeed regardless of network outcome. Otherwise a broken // kubectl/CNI fixture could masquerade as the expected denied connection. - target := fmt.Sprintf("http://%s:%d/", ip, port) + isolationAttempt(t, ns, source, allowed, `wget -q -T 2 -O /dev/null "$1"`, fmt.Sprintf("http://%s:%d/", ip, port)) +} + +// Each command is fixed by the test. Pass hosts, ports and URLs separately. +func isolationAttempt(t *testing.T, ns, source string, allowed bool, command string, args ...string) { + t.Helper() envInstance.Eventually(t, 45*time.Second, func() (bool, string) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - out, err := envInstance.Kubectl(ctx, "exec", "-n", "gameplane-games", source, "--", - "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) + execArgs := []string{"exec", "-n", ns, source, "--", "sh", "-c", "if " + command + "; then echo ALLOWED; else echo DENIED; fi", "probe"} + out, err := envInstance.Kubectl(ctx, append(execArgs, args...)...) if err != nil { return false, "probe execution failed: " + err.Error() + out } @@ -189,7 +382,7 @@ func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { if allowed { want = "ALLOWED" } - return got == want, fmt.Sprintf("%s -> %s: got %q, want %s; output: %q", source, target, got, want, out) + return got == want, fmt.Sprintf("%s -> %v: got %q, want %s; output: %q", source, args, got, want, out) }) } diff --git a/test/e2e/network_peers_e2e_test.go b/test/e2e/network_peers_e2e_test.go new file mode 100644 index 000000000..f36ef09b6 --- /dev/null +++ b/test/e2e/network_peers_e2e_test.go @@ -0,0 +1,181 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/hex" + "fmt" + "net/netip" + "os" + "os/exec" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +// The wrapper discovers both endpoints on kind's private IPv4 bridge. Render +// the parsed address before passing it to Helm, kubectl or Docker. +func kindFixtureIP(t *testing.T, key string) string { + t.Helper() + ip, err := netip.ParseAddr(os.Getenv(key)) + if err != nil || !ip.Is4() || !ip.IsPrivate() { + t.Fatalf("%s must identify a private IPv4 address on the kind bridge", key) + } + return ip.String() +} + +func checkTenantEndpoints(t *testing.T, ns, tenant, server, localPeer, homeIP, awayIP, peerIP string, profileApplied time.Time) { + t.Helper() + monitor := isolationClient(t, "monitoring", server+"-monitor", nil) + const fetchPeer = `wget --no-check-certificate -q -T 3 -O - "$1" | grep -Fxq isolated-kind-peer` + const scrape = `wget -q -T 3 -O - "$1" | grep -q '^gameplane_agent_'` + const connect = `nc -w 3 "$1" "$2" %s: %v %s", url, err, out) + }) +} + +func enableIsolatedFRP(t *testing.T, ns, server string) { + t.Helper() + ctx := t.Context() + envInstance.ApplyYAML(t, "frps.yaml") + game, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, server, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + auth := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: server + "-frp", OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: server, UID: game.GetUID()}, + }}, StringData: map[string]string{"token": ocFrpsToken}} + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Create(ctx, auth, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + // This is the administrator-supplied provider exception documented for + // tenant isolation. Cilium evaluates the selected frps backend after DNAT. + port := intstr.FromInt32(7000) + policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: server + "-frp-provider"}, + Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": "gameplane-tunnel", "app.kubernetes.io/instance": server}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: []networkingv1.NetworkPolicyEgressRule{{To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ocNS}}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "e2e-frps"}}, + }}, Ports: []networkingv1.NetworkPolicyPort{{Port: &port, Protocol: ptr.To(corev1.ProtocolTCP)}}}}, + }} + if _, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ns).Create(ctx, policy, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.NetworkingV1().NetworkPolicies(ns).Delete(context.Background(), policy.Name, metav1.DeleteOptions{}) + }) + patch := []byte(fmt.Sprintf(`{"spec":{"networking":{"tunnel":{"enabled":true,"provider":"frp","credentialsSecretRef":{"name":%q},"frp":{"serverAddr":%q,"remotePorts":[{"name":"game","remotePort":30000}]}}}}}`, auth.Name, ocFrpsAddr)) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, server, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + out, err := envInstance.Kubectl(ctx, "rollout", "status", "-n", ocNS, "deployment/e2e-frps", "--timeout=2m") + if err != nil { + t.Fatalf("frps: %v %s", err, out) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, server, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return ocTunnelReady(got) + }) + peers, err := envInstance.K8s.CoreV1().Pods(ocNS).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=e2e-frps"}) + if err != nil || len(peers.Items) != 1 { + t.Fatalf("frps pods: %v %v", peers, err) + } + control := isolationClient(t, "monitoring", server+"-frp-player", nil) + isolationAttempt(t, "monitoring", control, true, `wget -q -T 3 -O - "$1" | grep -Fxq reachable`, "http://"+peers.Items[0].Status.PodIP+":30000/") +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 0936453e0..510102c67 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -10,8 +10,16 @@ The amd64 `network-isolation` bucket requires the Cilium fixture (`GAMEPLANE_E2E_CNI=cilium`) and strict chart CIDRs derived from the single-node cluster. It tests actual legacy versus isolated sentinel connections, an existing template's opt-in rollout without selector changes, HTTP kubelet -readiness, fresh agent heartbeats, and API-to-agent mTLS access. It uses one -admin login and runs separately from the kindnet shape-only checks. +readiness, fresh agent heartbeats, and API-to-agent mTLS access. The tenant +profile drill exercises same-named games in two Restricted namespaces, real +frp forwarding, relay-to-waker access and revocation, and per-game versus +shared probe ports. A player on the kind bridge connects through a NodePort +with its source address preserved. An operator-created Backup writes to a Restic REST +repository on the bridge. A monitoring client reads both agents' metrics. +The drill checks fresh heartbeats after policy rollout and denies lateral agent +and RCON connections, private HTTPS and a reachable link-local metadata fixture. +`deploy/kind/with-network-peers.sh` installs the fixture route only inside the +disposable node and removes it and its peer containers on exit. ### Restricted workload drill (#714) diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index f0567e2b9..3b5e78698 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -20,7 +20,8 @@ { "tests": "^(TestPanel_OfflineRecovery|TestAPIClient_DoContextResponseBody|TestAPIClient_ExtractIntField)$", "components": ["operator", "api", "agent"], "paths": ["test/e2e/testdata/recovery/"] }, - { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent", "tunnel"], + "paths": ["test/e2e/fixtures/frps.yaml"] }, { "tests": "^TestMonitoring_DiscoveryPerformanceAndPolicy$", "components": ["operator", "api", "agent", "tunnel"] }, { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, diff --git a/test/e2e/telemetry_e2e_helpers_test.go b/test/e2e/telemetry_e2e_helpers_test.go index cb488897a..3f52d3c68 100644 --- a/test/e2e/telemetry_e2e_helpers_test.go +++ b/test/e2e/telemetry_e2e_helpers_test.go @@ -252,15 +252,25 @@ func tlDo(ctx context.Context, c *APIClient, method, path string, body any) (int // helmUpgradeReuse runs `helm upgrade --reuse-values` with the given --set // values on the e2e release and waits for the rollout. func helmUpgradeReuse(ctx context.Context, t *testing.T, sets ...string) { + t.Helper() + helmUpgradeReuseJSON(ctx, t, nil, sets...) +} + +// Helm merges persisted release values on upgrade. Use --set-json when the +// replacement must remain an explicitly empty array through that merge. +func helmUpgradeReuseJSON(ctx context.Context, t *testing.T, jsonSets []string, sets ...string) { t.Helper() args := []string{"upgrade", "gameplane", filepath.Join("..", "..", "charts", "gameplane"), "--kube-context", envInstance.Context, "--namespace", "gameplane-system", "--reuse-values"} for _, set := range sets { args = append(args, "--set", set) } + for _, set := range jsonSets { + args = append(args, "--set-json", set) + } args = append(args, "--wait", "--timeout", "6m") if out, err := exec.CommandContext(ctx, "helm", args...).CombinedOutput(); err != nil { - t.Fatalf("helm upgrade %v failed: %v\n%s", sets, err, out) + t.Fatalf("helm upgrade %v (JSON values %v) failed: %v\n%s", sets, jsonSets, err, out) } }