From 95e7b60c6cc4e53951924befdf2de246a7ed35ee Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:01:31 -0500 Subject: [PATCH 1/6] fix(networking): enforce an opt-in tenant network profile in managed namespaces Signed-off-by: Baylem --- .../gameplane/templates/networkpolicies.yaml | 47 +++++++--- charts/gameplane/templates/operator.yaml | 3 + charts/gameplane/templates/validate.yaml | 28 ++++++ charts/gameplane/values.yaml | 8 ++ docs/monitoring.md | 12 ++- docs/security.md | 53 ++++++++++- operator/cmd/main.go | 4 + .../controller/gameserver_controller.go | 1 + .../internal/controller/gameserver_tunnel.go | 21 ++++- .../controller/gameserver_tunnel_test.go | 59 ++++++++++++ test/e2e/network_isolation_e2e_test.go | 93 ++++++++++++++----- 11 files changed, 284 insertions(+), 45 deletions(-) diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index ed2ae0a8a..fa2ed06b9 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -1,5 +1,12 @@ {{- if (eq (include "gameplane.operatorEnabled" .) "true") }} {{- if .Values.networkPolicies.enabled }} +{{- $namespaces := list .Values.gamesNamespace }} +{{- if .Values.networkPolicies.tenantIsolation }} +{{- $namespaces = include "gameplane.operatorGameNamespaces" . | fromJsonArray }} +{{- end }} +{{- range $namespace := $namespaces }} +{{- with $ }} +--- # Default-deny ingress in the games namespace. Individual GameServers # open only the ports declared in their template (and whatever the # operator surfaces via the backing Service / Ingress). @@ -7,7 +14,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny-ingress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: {} @@ -23,7 +30,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: default-deny-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: {} @@ -33,6 +40,10 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: kube-system + {{- if .Values.networkPolicies.tenantIsolation }} + podSelector: + matchLabels: { k8s-app: kube-dns } + {{- end }} ports: - { protocol: UDP, port: 53 } - { protocol: TCP, port: 53 } @@ -47,7 +58,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-agent-to-apiserver - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -78,7 +89,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-api-to-agent - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -110,7 +121,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-api-to-capture - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -150,7 +161,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-prometheus-to-agent - namespace: {{ $.Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" $ | nindent 4 }} spec: podSelector: @@ -168,7 +179,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-prometheus-to-tunnel - namespace: {{ $.Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" $ | nindent 4 }} spec: podSelector: @@ -199,7 +210,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-kubelet-probes - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -237,7 +248,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-game-public-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -279,7 +290,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-backup-restore-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -288,7 +299,9 @@ spec: policyTypes: [Egress] egress: - to: - - ipBlock: { cidr: 0.0.0.0/0 } + {{- range ($backupEgress.toCIDRs | default (list "0.0.0.0/0")) }} + - ipBlock: { cidr: {{ . | quote }} } + {{- end }} ports: {{- range $backupEgress.ports }} - { protocol: {{ .protocol }}, port: {{ .port }} } @@ -306,7 +319,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-ingress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -332,7 +345,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-apiserver - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -356,6 +369,7 @@ spec: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } --- +{{- if not .Values.networkPolicies.tenantIsolation }} # Legacy sentinel proxy allowance, ingress half. This grants ALL ports between # non-isolated sentinels and games. NetworkPolicies are additive; per-server # player ingress does not narrow it. Opted-in templates use operator policies. @@ -369,7 +383,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-game - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -397,7 +411,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-sentinel-to-game-egress - namespace: {{ .Values.gamesNamespace }} + namespace: {{ $namespace }} labels: {{- include "gameplane.labels" . | nindent 4 }} spec: podSelector: @@ -415,5 +429,8 @@ spec: - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } # Intentionally unrestricted for templates that have not opted in. {{- end }} +{{- end }} +{{- end }} +{{- end }} {{- end }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 8e9427327..e37849cbd 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -329,6 +329,9 @@ spec: - --tunnel-frp-image={{ include "gameplane.tunnelFrpImage" . }} - --tunnel-tailscale-image={{ include "gameplane.tunnelTailscaleImage" . }} - --tunnel-playit-image={{ include "gameplane.tunnelPlayitImage" . }} + {{- if .Values.networkPolicies.tenantIsolation }} + - --restrict-tunnel-egress + {{- end }} {{- if .Values.operator.configInitImage }} - --config-init-image={{ .Values.operator.configInitImage }} {{- end }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 580f8ae3d..a5247baa0 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,4 +1,32 @@ {{- $network := .Values.networkPolicies | default dict }} +{{- if $network.tenantIsolation }} +{{- if or (not $network.strict) (not ($network.probeIngress | default dict).enabled) $network.probePorts }} +{{- fail "networkPolicies.tenantIsolation requires strict=true, probeIngress.enabled=true and empty shared probePorts" }} +{{- end }} +{{- if ($network.gameEgress | default dict).enabled }} +{{- fail "networkPolicies.tenantIsolation requires gameEgress.enabled=false; grant approved game downloads with destination-specific policies" }} +{{- end }} +{{- $protected := list .Release.Namespace "default" "kube-system" "kube-public" "kube-node-lease" .Values.operator.controlPlaneNamespace .Values.operator.agentCASecretNamespace .Values.operator.moduleNamespace }} +{{- range $ns := include "gameplane.operatorGameNamespaces" . | fromJsonArray }} +{{- if has $ns $protected }} +{{- fail "networkPolicies.tenantIsolation requires dedicated game namespaces" }} +{{- end }} +{{- end }} +{{- range $cidr := concat ($network.apiServerCIDRs | default list) ($network.kubeletCIDRs | default list) }} +{{- if not (regexMatch "(/32|/128)$" (toString $cidr)) }} +{{- fail "networkPolicies.tenantIsolation requires individual API and kubelet host CIDRs" }} +{{- end }} +{{- end }} +{{- $backup := $network.backupEgress | default (dict "enabled" true) }} +{{- if and $backup.enabled (or (not $backup.ports) (not $backup.toCIDRs)) }} +{{- fail "networkPolicies.tenantIsolation requires explicit backupEgress.toCIDRs and ports, or backupEgress.enabled=false" }} +{{- end }} +{{- range $backup.toCIDRs }} +{{- if hasSuffix "/0" (toString .) }} +{{- fail "networkPolicies.tenantIsolation forbids unrestricted backup destinations" }} +{{- end }} +{{- end }} +{{- end }} {{- if $network.strict }} {{- if not $network.enabled }} {{- fail "networkPolicies.strict requires networkPolicies.enabled=true" }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 503c95481..7e7d00796 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -576,6 +576,10 @@ networkPolicies: # Fail closed unless actual kubelet source CIDRs/probe ports and apiserver # endpoint CIDRs are supplied. Defaults retain the legacy broad allowances. strict: false + # Opt in only after provisioning each managed namespace and its provider + # egress. Requires strict endpoints, per-server probes, and explicit backup + # destinations. See the tenant network profile in docs/security.md. + tenantIsolation: false # CIDRs that hold kubelet source IPs for liveness/readiness probes. # If unset, the chart falls back to RFC1918 + link-local ranges, # which cover most clusters without opening the public internet. @@ -629,6 +633,10 @@ networkPolicies: # store) as well as public. backupEgress: enabled: true + # Explicit repository destinations. The compatibility profile defaults + # to any IPv4 address on the configured ports. Tenant isolation requires + # a nonempty list (or backupEgress.enabled=false and administrator policies). + toCIDRs: [] # Ports opened for restic repository connections. Defaults cover # HTTPS-based backends (S3, B2, REST-over-TLS, Swift, Azure, GCS) # and SFTP. Add your repository's port if it uses neither, e.g. a diff --git a/docs/monitoring.md b/docs/monitoring.md index 83a22b300..6f6245947 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -39,11 +39,13 @@ on TCP 9090 and `app.kubernetes.io/name: gameplane-tunnel` on TCP 8091, with a namespace selector for the scraper. Restricted Prometheus egress policies must also allow those destinations, DNS and API discovery. -For enforced isolation, configure `networkPolicies.strict: true`, actual -`kubeletCIDRs` and `apiServerCIDRs`, and the game modules' exact `probePorts`. -The legacy broad kubelet allowance can otherwise admit other private-address -sources on metrics ports. Keep metrics ports out of the game probe allowance; -the dedicated scrape policies grant their access. The CI monitoring bucket +For enforced isolation, configure `networkPolicies.strict: true` with the +actual kubelet and API endpoints. With per-server probe ingress enabled, +leave `networkPolicies.probePorts: []`. Each game receives its own probe ports. +Every explicit shared port grants kubelets access to that port on **all** game +pods. If you disable per-server probe ingress, list the shared ports explicitly: +an empty list in that legacy mode opens every game-pod port to the kubelet CIDRs. +Use the dedicated scrape policies for metrics. The CI monitoring bucket uses Cilium and actual node/API addresses, tests a scraper in a separate namespace, revokes/restores the scrape policy, and checks an unapproved namespace cannot reach agent metrics. diff --git a/docs/security.md b/docs/security.md index bf2a0a202..c4d2ecc33 100644 --- a/docs/security.md +++ b/docs/security.md @@ -306,6 +306,53 @@ intended for local `kind` development where mTLS is overkill. ## NetworkPolicies +### Tenant network profile + +Pre-provision a dedicated namespace per tenant and apply the cluster's restricted +Pod Security Admission labels there. Upgrade each wake-enabled template to +`spec.sentinelIsolation: true`, and verify its advertised ports before switching +profiles. With `networkPolicies.tenantIsolation: true`, Helm owns the baseline +policies in the primary namespace and every namespace in `operator.gameNamespaces`. +Inventory policies with the same names before enabling this on an existing system. +Removing a namespace from the list can remove its Helm-owned policies. Drain +its games or hand policy ownership to the cluster administrator first. + +```yaml +operator: + gameNamespaces: [tenant-b] +networkPolicies: + tenantIsolation: true + strict: true + kubeletCIDRs: [192.0.2.10/32, 192.0.2.11/32] + apiServerCIDRs: [192.0.2.2/32] + probeIngress: {enabled: true} + probePorts: [] + gameEgress: {enabled: false} + backupEgress: + enabled: true + toCIDRs: [192.0.2.30/32] + ports: [{protocol: TCP, port: 443}] +``` + +Replace the example addresses with endpoints observed by the enforcing CNI +after Service translation. The profile omits legacy cross-game sentinel rules, +limits DNS to kube-dns-labeled pods in kube-system, and gives each tunnel egress +to its own game/waker on advertised ports. Provide scoped DNS allowances if +the cluster uses NodeLocal DNS. Open approved asset downloads explicitly. +Use a namespace-local policy for each relay's provider control/data endpoints. +Select its `app.kubernetes.io/name: gameplane-tunnel` and +`app.kubernetes.io/instance: ` labels. For Playit also allow the exact +Kubernetes API endpoint for address publication. Providers with dynamic +destinations require a provider-aware egress policy or a separately isolated +deployment. Check existing provider policies after the operator reconciles. + +Run positive joins, heartbeat, scrape and backup checks on the deployed CNI. +From an independent tenant, check the target Pod IP and Service for RCON and +agent access. Verify private HTTPS and metadata endpoints are unreachable. +Review additional policies in each namespace because their grants are additive. + +### Chart and operator policy rules + When `networkPolicies.enabled=true` (default) the chart applies: - `default-deny-ingress` — denies all ingress to all pods in the games @@ -359,9 +406,9 @@ When `networkPolicies.enabled=true` (default) the chart applies: - `allow-prometheus-to-agent` — opt-in (rendered only when `serviceMonitors.scrapeNamespaceSelector` is set) policy allowing Prometheus pods in the selected namespace to reach the agent's plain-HTTP metrics port - (TCP 9090). That port is unauthenticated HTTP, not mTLS, and is otherwise - already reachable from `kubeletCIDRs` via `allow-kubelet-probes` above - unless `networkPolicies.probePorts` is narrowed to exclude it. + (TCP 9090). The per-server probe policy opens it only for a game that uses + that probe port. A shared `probePorts` entry for 9090 or the legacy all-port + kubelet allowance can also grant access to this unauthenticated listener. - `allow-game-public-egress` — (enabled by default, gated by `networkPolicies.gameEgress.enabled`) allows game pods to reach the public internet for binary/asset/mod downloads. Set `networkPolicies.gameEgress.enabled: false` diff --git a/operator/cmd/main.go b/operator/cmd/main.go index c52857639..b6955f51c 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -129,6 +129,7 @@ func main() { tunnelFrpImage string tunnelTailscaleImage string tunnelPlayitImage string + restrictTunnelEgress bool agentLogLevel string agentCABundle string agentClientCert string @@ -206,6 +207,8 @@ func main() { flag.StringVar(&tunnelTailscaleImage, "tunnel-tailscale-image", controller.DefaultTunnelTailscaleImage, "Image for the Tailscale tunnel relay pod that routes players through a Tailscale tailnet. "+ "Point at a private registry mirror for air-gapped installs.") + flag.BoolVar(&restrictTunnelEgress, "restrict-tunnel-egress", false, + "Only grant tunnel egress to this server's advertised backend ports; administrators supply provider egress policies.") flag.StringVar(&tunnelPlayitImage, "tunnel-playit-image", controller.DefaultTunnelPlayitImage, "Image for the Playit tunnel relay pod that routes players through a Playit tunnel. "+ "Point at a private registry mirror for air-gapped installs.") @@ -481,6 +484,7 @@ func main() { TunnelFrpImage: tunnelFrpImage, TunnelTailscaleImage: tunnelTailscaleImage, TunnelPlayitImage: tunnelPlayitImage, + RestrictTunnelEgress: restrictTunnelEgress, AgentLogLevel: agentLogLevel, AgentCASecretName: agentCASecretName, AgentCASecretNamespace: agentCASecretNamespace, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 536ae7bbb..02e4b0049 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -99,6 +99,7 @@ type GameServerReconciler struct { TunnelFrpImage string TunnelTailscaleImage string TunnelPlayitImage string + RestrictTunnelEgress bool // AgentCASecretName / AgentCASecretNamespace point at the cluster- // wide Secret holding `ca.crt` + `ca.key` used to sign the diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 7764240c8..22a48ec4f 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -582,6 +582,26 @@ func (r *GameServerReconciler) reconcileTunnelNetworkPolicy( }, } + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} + if r.RestrictTunnelEgress { + // DNS and provider endpoints get separate administrator-owned rules. + // An empty Ports list would grant every backend port. + np.Spec.Egress = nil + if ports := networkPolicyPortsFromTemplate(tmpl); len(ports) > 0 { + np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/instance": gs.Name}, + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: "app.kubernetes.io/name", Operator: metav1.LabelSelectorOpIn, + Values: []string{"gameplane-game", "gameplane-waker"}, + }}, + }}}, + Ports: ports, + }} + } + return controllerutil.SetControllerReference(gs, np, r.Scheme) + } + // Egress rules: allow DNS, advertised ports for inbound relay traffic, // and the provider's relay endpoint ports for outbound control traffic. egressPorts := make([]networkingv1.NetworkPolicyPort, 0) @@ -665,7 +685,6 @@ func (r *GameServerReconciler) reconcileTunnelNetworkPolicy( } // Egress with no "To" means "to any destination". - np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} egressRules := []networkingv1.NetworkPolicyEgressRule{{ Ports: egressPorts, // Empty To = allow to any destination diff --git a/operator/internal/controller/gameserver_tunnel_test.go b/operator/internal/controller/gameserver_tunnel_test.go index 2b23a570e..096068c58 100644 --- a/operator/internal/controller/gameserver_tunnel_test.go +++ b/operator/internal/controller/gameserver_tunnel_test.go @@ -1,13 +1,72 @@ package controller import ( + "reflect" "testing" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) +func TestRestrictTunnelEgressReplacesLegacyPlayitRules(t *testing.T) { + scheme := testScheme(t) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + r := &GameServerReconciler{Scheme: scheme, Client: fake.NewClientBuilder().WithScheme(scheme).Build()} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "one"}} + gs.Spec.Networking.Tunnel = &gameplanev1alpha1.GameServerTunnel{Provider: "playit"} + tmpl := singlePortTemplate(true, "minecraft") + reconcile := func() []networkingv1.NetworkPolicyEgressRule { + t.Helper() + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{wantTunnel: true}); err != nil { + t.Fatal(err) + } + var np networkingv1.NetworkPolicy + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}, &np); err != nil { + t.Fatal(err) + } + return np.Spec.Egress + } + if rules := reconcile(); len(rules) != 2 { + t.Fatalf("unexpected compatibility rules: %+v", rules) + } + r.RestrictTunnelEgress = true + rules := reconcile() + if len(rules) != 1 || len(rules[0].To) != 1 || !reflect.DeepEqual(rules[0].Ports, networkPolicyPortsFromTemplate(tmpl)) { + t.Fatalf("unexpected restricted rules: %+v", rules) + } + peer := rules[0].To[0] + if peer.PodSelector == nil || peer.NamespaceSelector != nil || peer.IPBlock != nil { + t.Fatalf("expected a pod peer in the local namespace: %+v", peer) + } + sel, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + t.Fatal(err) + } + for _, target := range []struct { + name, instance string + allowed bool + }{ + {"gameplane-game", "alpha", true}, {"gameplane-waker", "alpha", true}, + {"gameplane-game", "beta", false}, {"gameplane-operator", "alpha", false}, + } { + if sel.Matches(labels.Set{"app.kubernetes.io/name": target.name, "app.kubernetes.io/instance": target.instance}) != target.allowed { + t.Fatalf("unexpected peer match for %+v", target) + } + } + tmpl.Spec.Ports = nil + if rules := reconcile(); len(rules) != 0 { + t.Fatalf("empty advertised port set grants egress: %+v", rules) + } +} + // TestBuildFrpRemotePortsConfig covers F-052: the BACKING_SERVICE_PORT env // var must carry the template port's own containerPort and protocol // alongside the operator-chosen public remotePort, not just the remotePort diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 412b9b6ab..6eb99610d 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -33,18 +33,18 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { for _, pair := range [][2]string{{a, legacyTemplate}, {b, legacyTemplate}, {c, isolatedTemplate}} { applyBusyboxGameServer(t, ns, pair[0], pair[1]) } - aIP := isolationGameIP(t, a, false) - bIP := isolationGameIP(t, b, false) - cIP := isolationGameIP(t, c, true) - probeA := isolationProbe(t, a, false) - probeC := isolationProbe(t, c, true) + aIP := isolationGameIP(t, ns, a, false) + bIP := isolationGameIP(t, ns, b, false) + cIP := isolationGameIP(t, ns, c, true) + probeA := isolationProbe(t, ns, a, false) + probeC := isolationProbe(t, ns, c, true) // Compatibility: absent-field templates keep their existing broad grant. - isolationHTTP(t, probeA, bIP, 25575, true) - isolationHTTP(t, probeC, cIP, 25566, true) - isolationHTTP(t, probeC, cIP, 25575, false) - isolationHTTP(t, probeC, aIP, 25566, false) - isolationHTTP(t, probeA, cIP, 25566, false) + isolationHTTP(t, ns, probeA, bIP, 25575, true) + isolationHTTP(t, ns, probeC, cIP, 25566, true) + isolationHTTP(t, ns, probeC, cIP, 25575, false) + isolationHTTP(t, ns, probeC, aIP, 25566, false) + isolationHTTP(t, ns, probeA, cIP, 25566, false) ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) if err != nil { @@ -56,7 +56,7 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { if err != nil { t.Fatal(err) } - aIP, bIP = isolationGameIP(t, a, true), isolationGameIP(t, b, true) + aIP, bIP = isolationGameIP(t, ns, a, true), isolationGameIP(t, ns, b, true) ss, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) if err != nil || !reflect.DeepEqual(oldSelector, ss.Spec.Selector) { t.Fatalf("opt-in must preserve the immutable StatefulSet selector: %v", err) @@ -66,10 +66,10 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { if err != nil { t.Fatal(err) } - isolationHTTP(t, probeA, aIP, 25566, true) - isolationHTTP(t, probeA, aIP, 25575, false) - isolationHTTP(t, probeA, bIP, 25566, false) - isolationHTTP(t, probeA, bIP, 25575, false) + isolationHTTP(t, ns, probeA, aIP, 25566, true) + isolationHTTP(t, ns, probeA, aIP, 25575, false) + isolationHTTP(t, ns, probeA, bIP, 25566, false) + isolationHTTP(t, ns, probeA, bIP, 25575, false) // Kubelet HTTP readiness and agent heartbeats still work with strict CIDRs. // Also exercise an actual API -> agent mTLS call through its Service. @@ -87,6 +87,58 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { stamp, err := time.Parse(time.RFC3339, hb) return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" }) + + checkTenantNetworkProfile(t, ns, a, probeA) +} + +func checkTenantNetworkProfile(t *testing.T, ns, a, probeA string) { + t.Helper() + ctx := t.Context() + stamp := time.Now().UnixNano() + // Exercise the opt-in upgrade after checking the historical compatibility + // path. Identically named servers in separate namespaces must not become + // each other's relay backend. + tenant := fmt.Sprintf("tenant-net-%d", stamp) + _, err := envInstance.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: tenant}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) + }) + helmUpgradeReuse(ctx, t, "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", + "networkPolicies.probePorts=null", "networkPolicies.backupEgress.enabled=false", "networkPolicies.gameEgress.enabled=false") + secondTemplate := fmt.Sprintf("second-probe-%d", stamp) + isolationTemplate(t, secondTemplate, true) + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, secondTemplate, types.MergePatchType, + []byte(`{"spec":{"probes":{"readiness":{"httpGet":{"port":25575}}}}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, tenant, a, secondTemplate) + otherIP := isolationGameIP(t, tenant, a, true) + aIP := isolationGameIP(t, ns, a, true) + otherProbe := isolationProbe(t, tenant, a, true) + isolationHTTP(t, tenant, otherProbe, otherIP, 25566, true) + isolationHTTP(t, ns, probeA, otherIP, 25575, false) + isolationHTTP(t, tenant, otherProbe, aIP, 25575, false) + // Both Services resolve and are reachable from their own sentinels. + // Cross-tenant denials through the same DNS names exercise Service NAT. + home := a + "." + ns + ".svc" + away := a + "." + tenant + ".svc" + isolationHTTP(t, ns, probeA, home, 25566, true) + isolationHTTP(t, tenant, otherProbe, away, 25566, true) + isolationHTTP(t, ns, probeA, away, 25566, false) + isolationHTTP(t, tenant, otherProbe, home, 25566, false) + for namespace, want := range map[string]int{ns: 25566, tenant: 25575} { + policy, err := envInstance.K8s.NetworkingV1().NetworkPolicies(namespace).Get(ctx, a+"-probes", metav1.GetOptions{}) + if err != nil || len(policy.Spec.Ingress) != 1 || len(policy.Spec.Ingress[0].Ports) != 1 || + policy.Spec.Ingress[0].Ports[0].Port == nil || policy.Spec.Ingress[0].Ports[0].Port.IntValue() != want { + t.Fatalf("expected only kubelet probe port %d in %s: %v", want, namespace, err) + } + } } func isolationTemplate(t *testing.T, name string, isolated bool) { @@ -117,11 +169,11 @@ func isolationTemplate(t *testing.T, name string, isolated bool) { }) } -func isolationGameIP(t *testing.T, name string, isolated bool) string { +func isolationGameIP(t *testing.T, ns, name string, isolated bool) string { t.Helper() var ip string envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { - pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name+"-0", metav1.GetOptions{}) if err != nil { return false, err.Error() } @@ -141,9 +193,8 @@ func isolationGameIP(t *testing.T, name string, isolated bool) string { // A synthetic client carries the exact real sentinel selectors. It exposes no // listener and owns no Service/Deployment, so the running game is unaffected. -func isolationProbe(t *testing.T, server string, isolated bool) string { +func isolationProbe(t *testing.T, ns, server string, isolated bool) string { t.Helper() - const ns = "gameplane-games" name := server + "-probe" labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} if isolated { @@ -170,7 +221,7 @@ func isolationProbe(t *testing.T, server string, isolated bool) string { return name } -func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { +func isolationHTTP(t *testing.T, ns, source, ip string, port int, allowed bool) { t.Helper() // Every exec must succeed regardless of network outcome. Otherwise a broken // kubectl/CNI fixture could masquerade as the expected denied connection. @@ -178,7 +229,7 @@ func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { envInstance.Eventually(t, 45*time.Second, func() (bool, string) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - out, err := envInstance.Kubectl(ctx, "exec", "-n", "gameplane-games", source, "--", + out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, source, "--", "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) if err != nil { return false, "probe execution failed: " + err.Error() + out From 395ea08e228cc2a1425b09d74725fe2424fdbb9f Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 21:21:57 -0500 Subject: [PATCH 2/6] fix(networking): admit tunnel traffic at its own backend Signed-off-by: Baylem --- charts/gameplane/templates/validate.yaml | 18 +- charts/tests/check_network_isolation.py | 26 ++ docs/security.md | 6 +- .../controller/gameserver_controller.go | 8 +- .../internal/controller/gameserver_tunnel.go | 253 ++++++++---------- .../controller/gameserver_tunnel_test.go | 49 ++++ operator/internal/controller/owned_delete.go | 5 +- .../internal/controller/owned_delete_test.go | 5 + .../internal/controller/sentinel_isolation.go | 2 + .../sentinel_isolation_review_test.go | 2 +- operator/specs.md | 1 + test/e2e/network_isolation_e2e_test.go | 90 ++++++- test/e2e/specs.md | 4 +- 13 files changed, 304 insertions(+), 165 deletions(-) diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index a5247baa0..4089ba3f3 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -45,16 +45,24 @@ {{- $ipv4 := printf "^%s(\\.%s){3}/([0-9]|[12][0-9]|3[0-2])$" $octet $octet }} {{- $h := "[0-9A-Fa-f]{1,4}" }} {{- $ipv6 := printf "^((%s:){7}%s|(%s:){1,7}:|(%s:){1,6}:%s|(%s:){1,5}(:%s){1,2}|(%s:){1,4}(:%s){1,3}|(%s:){1,3}(:%s){1,4}|(%s:){1,2}(:%s){1,5}|%s:((:%s){1,6})|:((:%s){1,7}|:))/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h }} -{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" }} -{{- range (index $network $key) }} +{{- $cidrLists := dict "kubeletCIDRs" $network.kubeletCIDRs "apiServerCIDRs" $network.apiServerCIDRs }} +{{- $portLists := dict "probePorts" $network.probePorts }} +{{- if and $network.tenantIsolation ($network.backupEgress | default dict).enabled }} +{{- $_ := set $cidrLists "backupEgress.toCIDRs" $network.backupEgress.toCIDRs }} +{{- $_ := set $portLists "backupEgress.ports" $network.backupEgress.ports }} +{{- end }} +{{- range $key, $entries := $cidrLists }} +{{- range $entries }} {{- if or (not (kindIs "string" .)) (not (or (regexMatch $ipv4 (toString .)) (regexMatch $ipv6 (toString .)))) }} {{- fail (printf "networkPolicies.%s entries must be explicit IPv4/IPv6 CIDRs" $key) }} {{- end }} {{- end }} {{- end }} -{{- range $network.probePorts }} -{{- if or (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} -{{- fail "networkPolicies.probePorts requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" }} +{{- range $key, $entries := $portLists }} +{{- range $entries }} +{{- if or (not (kindIs "map" .)) (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} +{{- fail (printf "networkPolicies.%s requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" $key) }} +{{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/tests/check_network_isolation.py b/charts/tests/check_network_isolation.py index c8c267b7e..3200bf267 100644 --- a/charts/tests/check_network_isolation.py +++ b/charts/tests/check_network_isolation.py @@ -69,6 +69,32 @@ def main(): "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"][0] assert extra["ports"] == [{"protocol": "TCP", "port": 9000}] + tenant = copy.deepcopy(strict) + tenant["operator"] = {"gameNamespaces": ["tenant-a", "tenant-a"]} + tenant["networkPolicies"].update({ + "tenantIsolation": True, + "gameEgress": {"enabled": False}, + "backupEgress": {"enabled": True, "toCIDRs": ["192.0.2.3/32"], + "ports": [{"protocol": "TCP", "port": 443}]}, + }) + policies = [o for o in render(tenant) if o.get("kind") == "NetworkPolicy"] + for ns in ("gameplane-games", "tenant-a"): + local = [p for p in policies if p["metadata"]["namespace"] == ns] + names = [p["metadata"]["name"] for p in local] + assert len(names) == len(set(names)) + assert {"default-deny-ingress", "default-deny-egress", "allow-agent-to-apiserver"} <= set(names) + assert not {"allow-sentinel-to-game", "allow-sentinel-to-game-egress", "allow-kubelet-probes"} & set(names) + backup = one(local, "NetworkPolicy", "allow-backup-restore-egress") + assert backup["spec"]["egress"] == [{"to": [{"ipBlock": {"cidr": "192.0.2.3/32"}}], + "ports": [{"protocol": "TCP", "port": 443}]}] + for field, invalid_value, failure in (("ports", [{}], "integer port"), + ("ports", [{"protocol": "TCP", "port": 0}], "integer port"), + ("toCIDRs", ["192.0.2.999/32"], "must be explicit"), + ("toCIDRs", ["::/0"], "unrestricted backup")): + invalid = copy.deepcopy(tenant) + invalid["networkPolicies"]["backupEgress"][field] = invalid_value + render(invalid, failure=failure) + if __name__ == "__main__": main() diff --git a/docs/security.md b/docs/security.md index c4d2ecc33..0e4f6de31 100644 --- a/docs/security.md +++ b/docs/security.md @@ -318,6 +318,8 @@ Removing a namespace from the list can remove its Helm-owned policies. Drain its games or hand policy ownership to the cluster administrator first. ```yaml +podSecurity: + enforceRestricted: true operator: gameNamespaces: [tenant-b] networkPolicies: @@ -336,8 +338,8 @@ networkPolicies: Replace the example addresses with endpoints observed by the enforcing CNI after Service translation. The profile omits legacy cross-game sentinel rules, -limits DNS to kube-dns-labeled pods in kube-system, and gives each tunnel egress -to its own game/waker on advertised ports. Provide scoped DNS allowances if +limits DNS to kube-dns-labeled pods in kube-system, and admits both tunnel egress +and backend ingress between a relay and its own game/waker on advertised ports. Provide scoped DNS allowances if the cluster uses NodeLocal DNS. Open approved asset downloads explicitly. Use a namespace-local policy for each relay's provider control/data endpoints. Select its `app.kubernetes.io/name: gameplane-tunnel` and diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 02e4b0049..563c48345 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -436,14 +436,14 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // it will advertise. Like planSentinel, this is computed once per pass so // there is a single source of truth (see planTunnel's doc comment). tunnelPlan := r.planTunnel(ctx, &gs, &tmpl) - if err := r.reconcileTunnel(ctx, &gs, &tmpl, tunnelPlan.wantTunnel); err != nil { - logger.Error(err, "reconcile tunnel") - return ctrl.Result{}, err - } if err := r.reconcileTunnelNetworkPolicy(ctx, &gs, &tmpl, tunnelPlan); err != nil { logger.Error(err, "reconcile tunnel NetworkPolicy") return ctrl.Result{}, err } + if err := r.reconcileTunnel(ctx, &gs, &tmpl, tunnelPlan.wantTunnel); err != nil { + logger.Error(err, "reconcile tunnel") + return ctrl.Result{}, err + } if err := r.reconcileService(ctx, &gs, &tmpl, plan.routeToSentinel); err != nil { logger.Error(err, "reconcile Service") return ctrl.Result{}, err diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 22a48ec4f..8e2641c10 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "regexp" "strings" @@ -543,164 +544,128 @@ func (r *GameServerReconciler) clearTunnelCredentialRefused(ctx context.Context, return nil } -// reconcileTunnelNetworkPolicy maintains a per-server NetworkPolicy admitting -// the tunnel pod's outbound traffic. The games namespace runs a default-deny- -// egress policy (allow-game-public-egress selects app.kubernetes.io/name=gameplane-game), -// which would silently drop the tunnel pod's connection to the relay without -// an explicit admit rule. Without this, the tunnel feature silently does nothing. -// -// The policy admits egress to any destination on the advertised ports and DNS. +// The restricted profile admits both halves of the same-namespace relay path. +// Legacy provider/DNS egress remains available when the profile is disabled. func (r *GameServerReconciler) reconcileTunnelNetworkPolicy( ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, plan tunnelPlan, ) error { - np := &networkingv1.NetworkPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: gs.Name + "-tunnel-egress", - Namespace: gs.Namespace, - }, - } - - if !plan.wantTunnel { - // Converge-to-absent: only delete if we own it. - var existing networkingv1.NetworkPolicy - if err := r.Get(ctx, client.ObjectKeyFromObject(np), &existing); err != nil { - return client.IgnoreNotFound(err) - } - if !metav1.IsControlledBy(&existing, gs) { - return nil + tunnel := metav1.LabelSelector{MatchLabels: map[string]string{ + tunnelLabel: tunnelValue, "app.kubernetes.io/instance": gs.Name, + }} + backend := metav1.LabelSelector{ + MatchLabels: map[string]string{"app.kubernetes.io/instance": gs.Name}, + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: tunnelLabel, Operator: metav1.LabelSelectorOpIn, Values: []string{"gameplane-game", "gameplane-waker"}, + }}, + } + var egress, ingress *networkingv1.NetworkPolicySpec + if plan.wantTunnel { + egress = &networkingv1.NetworkPolicySpec{PodSelector: tunnel, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}} + ports := networkPolicyPortsFromTemplate(tmpl) + if !r.RestrictTunnelEgress { + egress.Egress = legacyTunnelEgress(gs.Spec.Networking.Tunnel, ports) + } else if len(ports) > 0 { + egress.Egress = []networkingv1.NetworkPolicyEgressRule{{ + To: []networkingv1.NetworkPolicyPeer{{PodSelector: &backend}}, Ports: ports, + }} + ingress = &networkingv1.NetworkPolicySpec{ + PodSelector: backend, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, + Ingress: []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &tunnel}}, Ports: ports, + }}, + } } - return client.IgnoreNotFound(r.Delete(ctx, &existing)) } + return errors.Join( + r.applyTunnelPolicy(ctx, gs, "-tunnel-egress", egress), + r.applyTunnelPolicy(ctx, gs, "-tunnel-ingress", ingress), + ) +} +func (r *GameServerReconciler) applyTunnelPolicy(ctx context.Context, gs *gameplanev1alpha1.GameServer, suffix string, spec *networkingv1.NetworkPolicySpec) error { + np := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + suffix, Namespace: gs.Namespace}} + if spec == nil { + return r.deleteIfControlledBy(ctx, gs, np) + } _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { - // Select the tunnel pod's labels. - np.Spec.PodSelector = metav1.LabelSelector{ - MatchLabels: map[string]string{ - tunnelLabel: tunnelValue, - "app.kubernetes.io/instance": gs.Name, - }, + if np.ResourceVersion != "" && !metav1.IsControlledBy(np, gs) { + return fmt.Errorf("tunnel policy %s/%s is not owned by this GameServer", np.Namespace, np.Name) } + np.Spec = *spec + return controllerutil.SetControllerReference(gs, np, r.Scheme) + }) + return err +} - np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} - if r.RestrictTunnelEgress { - // DNS and provider endpoints get separate administrator-owned rules. - // An empty Ports list would grant every backend port. - np.Spec.Egress = nil - if ports := networkPolicyPortsFromTemplate(tmpl); len(ports) > 0 { - np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{{ - To: []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"app.kubernetes.io/instance": gs.Name}, - MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: "app.kubernetes.io/name", Operator: metav1.LabelSelectorOpIn, - Values: []string{"gameplane-game", "gameplane-waker"}, - }}, - }}}, - Ports: ports, - }} - } - return controllerutil.SetControllerReference(gs, np, r.Scheme) - } +func legacyTunnelEgress(tunnel *gameplanev1alpha1.GameServerTunnel, advertised []networkingv1.NetworkPolicyPort) []networkingv1.NetworkPolicyEgressRule { + // Egress rules: allow DNS, advertised ports for inbound relay traffic, + // and the provider's relay endpoint ports for outbound control traffic. + egressPorts := make([]networkingv1.NetworkPolicyPort, 0) + + // DNS: UDP and TCP port 53 to any destination (required by all providers). + tcpProto := corev1.ProtocolTCP + udpProto := corev1.ProtocolUDP + dnsPort := intstr.FromInt(53) + egressPorts = append(egressPorts, + networkingv1.NetworkPolicyPort{ + Protocol: &udpProto, + Port: &dnsPort, + }, + networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &dnsPort, + }, + ) - // Egress rules: allow DNS, advertised ports for inbound relay traffic, - // and the provider's relay endpoint ports for outbound control traffic. - egressPorts := make([]networkingv1.NetworkPolicyPort, 0) - - // DNS: UDP and TCP port 53 to any destination (required by all providers). - tcpProto := corev1.ProtocolTCP - udpProto := corev1.ProtocolUDP - dnsPort := intstr.FromInt(53) - egressPorts = append(egressPorts, - networkingv1.NetworkPolicyPort{ - Protocol: &udpProto, - Port: &dnsPort, - }, - networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &dnsPort, - }, - ) - - // Provider relay egress: the ports the tunnel process dials outbound. - tunnel := gs.Spec.Networking.Tunnel - if tunnel != nil { - switch tunnel.Provider { - case "frp": - // frp: outbound to the frp server on ServerAddr:ServerPort (default 7000). - if tunnel.Frp != nil { - serverPort := tunnel.Frp.ServerPort - if serverPort == 0 { - serverPort = 7000 - } - port := intstr.FromInt32(serverPort) - egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &port, - }) + // Provider relay egress: the ports the tunnel process dials outbound. + if tunnel != nil { + switch tunnel.Provider { + case "frp": + // frp: outbound to the frp server on ServerAddr:ServerPort (default 7000). + if tunnel.Frp != nil { + serverPort := tunnel.Frp.ServerPort + if serverPort == 0 { + serverPort = 7000 } - - case "tailscale": - // tailscale: outbound to control plane on TCP 443 and DERP relays on UDP 41641. - httpsPort := intstr.FromInt(443) - derpPort := intstr.FromInt(41641) - egressPorts = append(egressPorts, - networkingv1.NetworkPolicyPort{ - Protocol: &tcpProto, - Port: &httpsPort, - }, - networkingv1.NetworkPolicyPort{ - Protocol: &udpProto, - Port: &derpPort, - }, - ) - - case "playit": - // playit does not publish a fixed set of relay endpoints or ports: - // the playit agent dials its control plane and relay nodes on - // varying TCP and UDP ports that are not known ahead of time. - // A per-port allow list (like frp's ServerPort or tailscale's - // 443/41641) can't express that, so playit gets its own egress - // rule below with no Ports field at all -- which, per the K8s - // NetworkPolicy semantics, means all ports/protocols -- to any - // destination, matching how the frp/tailscale rule above omits - // "To" to allow any destination. The DNS and advertised-port - // rule built here is left untouched for playit. - } - } - - // Advertised ports: the game's container ports (tunnel receives here, forwards inbound). - for _, p := range tmpl.Spec.Ports { - if !p.Advertise { - continue - } - proto := p.Protocol - if proto == "" { - proto = corev1.ProtocolTCP + port := intstr.FromInt32(serverPort) + egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &port, + }) } - port := intstr.FromInt32(p.ContainerPort) - egressPorts = append(egressPorts, networkingv1.NetworkPolicyPort{ - Protocol: &proto, - Port: &port, - }) - } - // Egress with no "To" means "to any destination". - egressRules := []networkingv1.NetworkPolicyEgressRule{{ - Ports: egressPorts, - // Empty To = allow to any destination - }} - if tunnel != nil && tunnel.Provider == "playit" { - // playit's relay/control-plane ports aren't statically known, so - // grant it a separate rule with no Ports (= all ports/protocols) - // and no To (= any destination), on top of the DNS/advertised-port - // rule above. - egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{}) - } - np.Spec.Egress = egressRules - - return controllerutil.SetControllerReference(gs, np, r.Scheme) - }) - return err + case "tailscale": + // tailscale: outbound to control plane on TCP 443 and DERP relays on UDP 41641. + httpsPort := intstr.FromInt(443) + derpPort := intstr.FromInt(41641) + egressPorts = append(egressPorts, + networkingv1.NetworkPolicyPort{ + Protocol: &tcpProto, + Port: &httpsPort, + }, + networkingv1.NetworkPolicyPort{ + Protocol: &udpProto, + Port: &derpPort, + }, + ) + + } + } + + // Egress with no "To" means "to any destination". + egressRules := []networkingv1.NetworkPolicyEgressRule{{ + Ports: append(egressPorts, advertised...), + // Empty To = allow to any destination + }} + if tunnel != nil && tunnel.Provider == "playit" { + // playit's relay/control-plane ports aren't statically known, so + // grant it a separate rule with no Ports (= all ports/protocols) + // and no To (= any destination), on top of the DNS/advertised-port + // rule above. + egressRules = append(egressRules, networkingv1.NetworkPolicyEgressRule{}) + } + return egressRules } // getTunnel fetches the tunnel Deployment, reporting exists=false when absent. diff --git a/operator/internal/controller/gameserver_tunnel_test.go b/operator/internal/controller/gameserver_tunnel_test.go index 096068c58..184969060 100644 --- a/operator/internal/controller/gameserver_tunnel_test.go +++ b/operator/internal/controller/gameserver_tunnel_test.go @@ -6,6 +6,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" @@ -61,10 +62,58 @@ func TestRestrictTunnelEgressReplacesLegacyPlayitRules(t *testing.T) { t.Fatalf("unexpected peer match for %+v", target) } } + var ingress networkingv1.NetworkPolicy + inKey := client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-ingress"} + if err := r.Get(t.Context(), inKey, &ingress); err != nil { + t.Fatal(err) + } + wantTunnel := metav1.LabelSelector{MatchLabels: map[string]string{ + tunnelLabel: tunnelValue, "app.kubernetes.io/instance": gs.Name, + }} + wantRules := []networkingv1.NetworkPolicyIngressRule{{ + From: []networkingv1.NetworkPolicyPeer{{PodSelector: &wantTunnel}}, Ports: rules[0].Ports, + }} + if !reflect.DeepEqual(ingress.Spec.PodSelector, *peer.PodSelector) || + !reflect.DeepEqual(ingress.Spec.Ingress, wantRules) || !metav1.IsControlledBy(&ingress, gs) { + t.Fatalf("unexpected ingress policy: %+v", ingress) + } + absent := func(key client.ObjectKey) { + t.Helper() + if err := r.Get(t.Context(), key, &networkingv1.NetworkPolicy{}); !apierrors.IsNotFound(err) { + t.Fatalf("expected %s to be absent, got %v", key, err) + } + } + r.RestrictTunnelEgress = false + reconcile() + absent(inKey) + r.RestrictTunnelEgress = true + reconcile() tmpl.Spec.Ports = nil if rules := reconcile(); len(rules) != 0 { t.Fatalf("empty advertised port set grants egress: %+v", rules) } + absent(inKey) + tmpl = singlePortTemplate(true, "minecraft") + reconcile() + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{}); err != nil { + t.Fatal(err) + } + absent(inKey) + absent(client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}) + + foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: inKey.Name, Namespace: inKey.Namespace, UID: "foreign"}} + if err := r.Create(t.Context(), foreign); err != nil { + t.Fatal(err) + } + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{wantTunnel: true}); err == nil { + t.Fatal("accepted foreign ingress policy") + } + if err := r.reconcileTunnelNetworkPolicy(t.Context(), gs, tmpl, tunnelPlan{}); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), inKey, &ingress); err != nil || ingress.UID != foreign.UID || len(ingress.OwnerReferences) != 0 { + t.Fatalf("changed foreign ingress policy: %+v, %v", ingress, err) + } } // TestBuildFrpRemotePortsConfig covers F-052: the BACKING_SERVICE_PORT env diff --git a/operator/internal/controller/owned_delete.go b/operator/internal/controller/owned_delete.go index 69845e5f7..84e9e340e 100644 --- a/operator/internal/controller/owned_delete.go +++ b/operator/internal/controller/owned_delete.go @@ -38,6 +38,7 @@ func (r *GameServerReconciler) deleteIfControlledBy( // the ownership check above and this Delete reaching the API server // would still match on name and be deleted, even though this GameServer // never controlled the replacement. - uid := obj.GetUID() - return client.IgnoreNotFound(r.Delete(ctx, obj, client.Preconditions{UID: &uid})) + // Also fence ownership changes to the same object between the read and delete. + uid, version := obj.GetUID(), obj.GetResourceVersion() + return client.IgnoreNotFound(r.Delete(ctx, obj, client.Preconditions{UID: &uid, ResourceVersion: &version})) } diff --git a/operator/internal/controller/owned_delete_test.go b/operator/internal/controller/owned_delete_test.go index 3dd8907d4..363460b10 100644 --- a/operator/internal/controller/owned_delete_test.go +++ b/operator/internal/controller/owned_delete_test.go @@ -190,6 +190,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { controllerutil.SetControllerReference(gs, secret, scheme) var gotUID *types.UID + var gotVersion *string cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gs, secret). WithInterceptorFuncs(interceptor.Funcs{ Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { @@ -197,6 +198,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { do.ApplyOptions(opts) if do.Preconditions != nil { gotUID = do.Preconditions.UID + gotVersion = do.Preconditions.ResourceVersion } return c.Delete(ctx, obj, opts...) }, @@ -213,4 +215,7 @@ func TestDeleteIfControlledBy_DeleteIsUIDPreconditioned(t *testing.T) { if *gotUID != secret.UID { t.Errorf("Delete UID precondition = %q, want %q", *gotUID, secret.UID) } + if gotVersion == nil || *gotVersion == "" || *gotVersion != secret.ResourceVersion { + t.Fatalf("expected delete fenced at observed resourceVersion %q, got %v", secret.ResourceVersion, gotVersion) + } } diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go index 20a1057a3..f76573b7e 100644 --- a/operator/internal/controller/sentinel_isolation.go +++ b/operator/internal/controller/sentinel_isolation.go @@ -133,6 +133,8 @@ func (r *GameServerReconciler) revokeInvalidSentinelIsolationExposure(ctx contex &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gameIngressPolicyName(gs)}}, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-ingress"}}, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-egress"}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-ingress"}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-tunnel-egress"}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-game-direct"}}, } diff --git a/operator/internal/controller/sentinel_isolation_review_test.go b/operator/internal/controller/sentinel_isolation_review_test.go index d3859e210..b9e3caf8d 100644 --- a/operator/internal/controller/sentinel_isolation_review_test.go +++ b/operator/internal/controller/sentinel_isolation_review_test.go @@ -120,7 +120,7 @@ func TestInvalidIsolatedTemplateRevokesOnlyOwnedExposure(t *testing.T) { owner.UID = "foreign-server" } var exposures []client.Object - for _, name := range []string{gameIngressPolicyName(gs), gs.Name + "-sentinel-ingress", gs.Name + "-sentinel-egress"} { + for _, name := range []string{gameIngressPolicyName(gs), gs.Name + "-sentinel-ingress", gs.Name + "-sentinel-egress", gs.Name + "-tunnel-ingress", gs.Name + "-tunnel-egress"} { exposures = append(exposures, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name), OwnerReferences: []metav1.OwnerReference{owner}}}) } for _, name := range []string{gs.Name, gs.Name + "-game-direct"} { diff --git a/operator/specs.md b/operator/specs.md index ad0254e01..1330a10a9 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -355,6 +355,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. + - Tunnel NetworkPolicies use the relay's provider ports in compatibility mode. With `--restrict-tunnel-egress`, a server's tunnel gets egress to its own game or waker and the backend gets the matching ingress from that tunnel. Both rules select only the same namespace, server instance and advertised port/protocol. The chart covers DNS and administrators supply provider egress. - Probe NetworkPolicy (`probe_policy.go`): with `--probe-ingress-policy`, reconcile an owned `-probes` policy admitting `--probe-ingress-from-cidr` to exactly the ports the game container's tcpSocket, httpGet (no `host`) and grpc liveness, readiness and startup probes use; named ports stay named. Exec probes need no network. With the flag off, no network probe, or no CIDRs, delete the policy if the GameServer controls it (a rule with no ports would admit every port). Replaces the chart's catch-all `allow-kubelet-probes` rule. - Template env ownership: Secret and ConfigMap references in the GameTemplate's and GameVersion's `env` must pass the same ownership rule as `spec.env` (`validateTemplateEnvSources` reuses `validateServerEnvSources`), so a template cannot mount another server's Secret into this server's pod. - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. An opted-in template with zero advertised ports revokes its owned game-ingress and sentinel policies plus public/game-direct Services through uncached, UID/resourceVersion-guarded cleanup before reporting actionable `Ready=False`; it creates no workloads or unrestricted grants, preserves foreign resources and the agent Service, and reports the failure even when cleanup needs a retry. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. On opt-out, roll the labels off both workloads and retain the per-server grants until uncached reads confirm both controllers observed the unlabeled templates and completed their rollouts, and neither game nor sentinel pods still carry the label. Settled opt-outs first probe cached sentinel policies and skip live rollout reads when no owned grants exist. The owned NetworkPolicy watch must remain enabled: delayed cache visibility enqueues another cleanup pass. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 6eb99610d..68770b0cd 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -11,6 +11,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -88,10 +89,10 @@ func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" }) - checkTenantNetworkProfile(t, ns, a, probeA) + checkTenantNetworkProfile(t, ns, a, b, probeA) } -func checkTenantNetworkProfile(t *testing.T, ns, a, probeA string) { +func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { t.Helper() ctx := t.Context() stamp := time.Now().UnixNano() @@ -109,7 +110,8 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, probeA string) { _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) }) helmUpgradeReuse(ctx, t, "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", - "networkPolicies.probePorts=null", "networkPolicies.backupEgress.enabled=false", "networkPolicies.gameEgress.enabled=false") + "networkPolicies.probePorts=null", "networkPolicies.backupEgress.enabled=false", "networkPolicies.gameEgress.enabled=false", + "networkPolicies.gameIngress.fromCIDRs[0]=192.0.2.1/32") secondTemplate := fmt.Sprintf("second-probe-%d", stamp) isolationTemplate(t, secondTemplate, true) _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, secondTemplate, types.MergePatchType, @@ -139,6 +141,79 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, probeA string) { t.Fatalf("expected only kubelet probe port %d in %s: %v", want, namespace, err) } } + + checkTunnelPolicy(t, ns, a, b, probeA, aIP, home, away) + checkSharedProbeFallback(t, ns, a, secondTemplate) +} + +func checkTunnelPolicy(t *testing.T, ns, a, b, probeA, aIP, home, away string) { + t.Helper() + ctx := t.Context() + // Enable a real GameServer tunnel with a fixture endpoint. A synthetic tunnel + // pod exercises its policy without needing upstream provider credentials. + enable := []byte(`{"spec":{"networking":{"tunnel":{"enabled":true,"provider":"frp","credentialsSecretRef":{"name":"fixture-token"},"frp":{"serverAddr":"192.0.2.1","remotePorts":[{"name":"game","remotePort":30000}]}}}}}`) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, a, types.MergePatchType, enable, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + tunnel := isolationClient(t, ns, a+"-relay-probe", map[string]string{ + "app.kubernetes.io/name": "gameplane-tunnel", "app.kubernetes.io/instance": a, + }) + isolationHTTP(t, ns, tunnel, aIP, 25566, true) + isolationHTTP(t, ns, tunnel, home, 25566, true) + isolationHTTP(t, ns, tunnel, aIP, 25575, false) + isolationHTTP(t, ns, tunnel, b+"."+ns+".svc", 25566, false) + isolationHTTP(t, ns, tunnel, away, 25566, false) + // Stand up a listener on the existing synthetic waker. The game is running, + // so its main service continues to route exclusively to its game pod. + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, probeA, "--", "sh", "-c", + "mkdir -p /tmp/wake && echo awake >/tmp/wake/index.html && httpd -p 25566 -h /tmp/wake"); err != nil { + t.Fatalf("start wake listener: %v: %s", err, out) + } + waker, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, probeA, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + isolationHTTP(t, ns, tunnel, waker.Status.PodIP, 25566, true) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, a, types.MergePatchType, + []byte(`{"spec":{"networking":{"tunnel":{"enabled":false}}}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + isolationHTTP(t, ns, tunnel, aIP, 25566, false) +} + +func checkSharedProbeFallback(t *testing.T, ns, a, secondTemplate string) { + t.Helper() + ctx := t.Context() + // Exercise the explicit shared-probe fallback in the primary namespace. + // Both servers have active readiness probes but need different ports. + fallbackGame := fmt.Sprintf("net-fallback-%d", time.Now().UnixNano()) + applyBusyboxGameServer(t, ns, fallbackGame, secondTemplate) + isolationGameIP(t, ns, fallbackGame, true) + helmUpgradeReuse(ctx, t, "networkPolicies.tenantIsolation=false", "networkPolicies.probeIngress.enabled=false", + "networkPolicies.probePorts[0].port=25566", "networkPolicies.probePorts[0].protocol=TCP", + "networkPolicies.probePorts[1].port=25575", "networkPolicies.probePorts[1].protocol=TCP") + policies := envInstance.K8s.NetworkingV1().NetworkPolicies(ns) + shared, err := policies.Get(ctx, "allow-kubelet-probes", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(shared.Spec.PodSelector.MatchLabels, map[string]string{"app.kubernetes.io/name": "gameplane-game"}) || + len(shared.Spec.Ingress) != 1 || len(shared.Spec.Ingress[0].Ports) != 2 { + t.Fatalf("unexpected shared kubelet rule: %+v", shared.Spec) + } + for i, want := range []int{25566, 25575} { + port := shared.Spec.Ingress[0].Ports[i] + if port.Protocol == nil || *port.Protocol != corev1.ProtocolTCP || port.Port == nil || port.Port.IntValue() != want { + t.Fatalf("unexpected shared probe port: %+v", port) + } + } + for _, server := range []string{a, fallbackGame} { + envInstance.Eventually(t, time.Minute, func() (bool, string) { + _, err := policies.Get(ctx, server+"-probes", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("per-game probe rule for %s still present: %v", server, err) + }) + isolationGameIP(t, ns, server, true) + } } func isolationTemplate(t *testing.T, name string, isolated bool) { @@ -191,15 +266,18 @@ func isolationGameIP(t *testing.T, ns, name string, isolated bool) string { return ip } -// A synthetic client carries the exact real sentinel selectors. It exposes no -// listener and owns no Service/Deployment, so the running game is unaffected. +// A synthetic client carries the real sentinel selectors in its game namespace. func isolationProbe(t *testing.T, ns, server string, isolated bool) string { t.Helper() - name := server + "-probe" labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} if isolated { labels["gameplane.local/sentinel-isolation"] = "true" } + return isolationClient(t, ns, server+"-probe", labels) +} + +func isolationClient(t *testing.T, ns, name string, labels map[string]string) string { + t.Helper() noToken := false pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 9caad3502..047e00451 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -10,7 +10,9 @@ The amd64 `network-isolation` bucket requires the Cilium fixture (`GAMEPLANE_E2E_CNI=cilium`) and strict chart CIDRs derived from the single-node cluster. It tests actual legacy versus isolated sentinel connections, an existing template's opt-in rollout without selector changes, HTTP kubelet -readiness, fresh agent heartbeats, and API-to-agent mTLS access. It uses one +readiness, fresh agent heartbeats, and API-to-agent mTLS access. The tenant +profile drill exercises same-named games in separate namespaces, relay-to-game +and relay-to-waker access, revocation, and per-game versus shared probe ports. It uses one admin login and runs separately from the kindnet shape-only checks. ### Restricted workload drill (#714) From 0a3bc8b574bdfe163867b32ebc76074be8c49319 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 23:24:18 -0500 Subject: [PATCH 3/6] test(networking): exercise tenant policies against live peers on Cilium Signed-off-by: Baylem --- .github/workflows/ci.yaml | 13 +- deploy/kind/with-network-peers.sh | 48 +++++++ test/e2e/network_isolation_e2e_test.go | 61 ++++++--- test/e2e/network_peers_e2e_test.go | 167 +++++++++++++++++++++++++ test/e2e/specs.md | 12 +- test/e2e/suite-components.json | 3 +- 6 files changed, 281 insertions(+), 23 deletions(-) create mode 100644 deploy/kind/with-network-peers.sh create mode 100644 test/e2e/network_peers_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ce0fa057..918a22b15 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1852,8 +1852,8 @@ jobs: job_timeout: 75 - bucket: network-isolation parallel: 1 - test_timeout: 15m - job_timeout: 60 + test_timeout: 25m + job_timeout: 75 - bucket: operator parallel: 8 test_timeout: 20m @@ -1939,17 +1939,22 @@ jobs: GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'wake-hostport' && 'two-node' || (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: install real monitoring fixture - if: matrix.bucket == 'monitoring' && env.RUN_REGEX != '' + if: (matrix.bucket == 'monitoring' || matrix.bucket == 'network-isolation') && env.RUN_REGEX != '' run: bash deploy/kind/monitoring.sh - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' env: GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports + E2E_BUCKET: ${{ matrix.bucket }} run: | + prefix=() + if [[ "$E2E_BUCKET" == network-isolation ]]; then + prefix=(bash ../../deploy/kind/with-network-peers.sh) + fi cd test/e2e && \ GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ - go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ + "${prefix[@]}" go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./... - name: upload measured panel recovery report if: always() && matrix.bucket == 'recovery' diff --git a/deploy/kind/with-network-peers.sh b/deploy/kind/with-network-peers.sh new file mode 100644 index 000000000..32ed7d800 --- /dev/null +++ b/deploy/kind/with-network-peers.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Run the Cilium bucket with reachable, off-cluster endpoints on kind's bridge. +set -euo pipefail +node="$(kind get nodes --name "${CLUSTER:?}" | head -n1)" +[[ -n "$node" && "$(docker inspect --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}' "$node")" == "$CLUSTER" ]] +scratch="$(mktemp -d)" +peer="" repo="" route_added=false +metadata=169.254.169.253 +cleanup() { + if [[ "$route_added" == true ]]; then docker exec "$node" ip route del "$metadata/32" via "$GAMEPLANE_NET_PEER_IP" || true; fi + if [[ -n "$repo" ]]; then docker rm -f "$repo" >/dev/null || true; fi + if [[ -n "$peer" ]]; then docker rm -f "$peer" >/dev/null || true; fi + rm -rf "$scratch" +} +trap cleanup EXIT +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj /CN=kind-private-fixture \ + -keyout "$scratch/tls.key" -out "$scratch/tls.crt" >/dev/null 2>&1 +cat >"$scratch/listen.py" <<'PY' +import http.server, ssl, threading + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"isolated-kind-peer\n") + +metadata = http.server.ThreadingHTTPServer(("169.254.169.253", 80), Handler) +threading.Thread(target=metadata.serve_forever, daemon=True).start() +tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +tls.load_cert_chain("/fixture/tls.crt", "/fixture/tls.key") +https = http.server.ThreadingHTTPServer(("0.0.0.0", 443), Handler) +https.socket = tls.wrap_socket(https.socket, server_side=True) +https.serve_forever() +PY +peer="$(docker run --detach --network kind --cap-add NET_ADMIN --volume "$scratch:/fixture:ro" \ + python:3.14-alpine sh -ec 'ip addr add 169.254.169.253/32 dev lo; exec python /fixture/listen.py')" +GAMEPLANE_NET_PEER_IP="$(docker inspect --format '{{(index .NetworkSettings.Networks "kind").IPAddress}}' "$peer")" +[[ -n "$GAMEPLANE_NET_PEER_IP" ]] +# Route only this alternate metadata address inside the disposable node. +# The real cloud metadata address is never used. +docker exec "$node" ip route add "$metadata/32" via "$GAMEPLANE_NET_PEER_IP" +route_added=true +repo="$(docker run --detach --network "container:$peer" --entrypoint rest-server \ + restic/rest-server:0.14.0 --listen :8000 --no-auth --path /tmp/repository)" +GAMEPLANE_NET_NODE_IP="$(docker inspect --format '{{(index .NetworkSettings.Networks "kind").IPAddress}}' "$node")" +export GAMEPLANE_NET_PEER_IP GAMEPLANE_NET_NODE_IP +export GAMEPLANE_NET_PEER_CONTAINER="$peer" +"$@" diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 68770b0cd..d42646ee6 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -5,6 +5,8 @@ package e2e import ( "context" "fmt" + "net" + "os" "reflect" "strings" "testing" @@ -15,6 +17,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" ) // This bucket uses Cilium, not kindnet. Exercise the effective union of all @@ -101,7 +104,9 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { // each other's relay backend. tenant := fmt.Sprintf("tenant-net-%d", stamp) _, err := envInstance.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{Name: tenant}, + ObjectMeta: metav1.ObjectMeta{Name: tenant, Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "restricted", "pod-security.kubernetes.io/enforce-version": "v1.34", + }}, }, metav1.CreateOptions{}) if err != nil { t.Fatal(err) @@ -109,9 +114,16 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { t.Cleanup(func() { _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) }) + peerIP := os.Getenv("GAMEPLANE_NET_PEER_IP") + if ip := net.ParseIP(peerIP); ip == nil || ip.To4() == nil || !ip.IsPrivate() { + t.Fatal("network isolation requires the private IPv4 endpoint from deploy/kind/with-network-peers.sh") + } helmUpgradeReuse(ctx, t, "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", - "networkPolicies.probePorts=null", "networkPolicies.backupEgress.enabled=false", "networkPolicies.gameEgress.enabled=false", - "networkPolicies.gameIngress.fromCIDRs[0]=192.0.2.1/32") + "podSecurity.enforceRestricted=true", "networkPolicies.probePorts=null", "networkPolicies.gameEgress.enabled=false", + "networkPolicies.backupEgress.enabled=true", "networkPolicies.backupEgress.toCIDRs[0]="+peerIP+"/32", + "networkPolicies.backupEgress.ports[0].port=8000", "networkPolicies.backupEgress.ports[0].protocol=TCP", + "networkPolicies.gameIngress.fromCIDRs[0]="+peerIP+"/32") + profileApplied := time.Now().UTC().Truncate(time.Second) secondTemplate := fmt.Sprintf("second-probe-%d", stamp) isolationTemplate(t, secondTemplate, true) _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, secondTemplate, types.MergePatchType, @@ -142,6 +154,7 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { } } + checkTenantEndpoints(t, ns, tenant, a, b, aIP, otherIP, peerIP, profileApplied) checkTunnelPolicy(t, ns, a, b, probeA, aIP, home, away) checkSharedProbeFallback(t, ns, a, secondTemplate) } @@ -149,12 +162,7 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { func checkTunnelPolicy(t *testing.T, ns, a, b, probeA, aIP, home, away string) { t.Helper() ctx := t.Context() - // Enable a real GameServer tunnel with a fixture endpoint. A synthetic tunnel - // pod exercises its policy without needing upstream provider credentials. - enable := []byte(`{"spec":{"networking":{"tunnel":{"enabled":true,"provider":"frp","credentialsSecretRef":{"name":"fixture-token"},"frp":{"serverAddr":"192.0.2.1","remotePorts":[{"name":"game","remotePort":30000}]}}}}}`) - if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, a, types.MergePatchType, enable, metav1.PatchOptions{}); err != nil { - t.Fatal(err) - } + enableIsolatedFRP(t, ns, a) tunnel := isolationClient(t, ns, a+"-relay-probe", map[string]string{ "app.kubernetes.io/name": "gameplane-tunnel", "app.kubernetes.io/instance": a, }) @@ -220,7 +228,8 @@ func isolationTemplate(t *testing.T, name string, isolated bool) { t.Helper() spec := map[string]any{ "displayName": name, "game": "busybox", "version": "1", "image": "busybox:1.36", - "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, + "security": map[string]any{"runAsUser": int64(1000), "runAsGroup": int64(1000), "fsGroup": int64(1000)}, + "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, "ports": []any{ map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}, map[string]any{"name": "management", "containerPort": int64(25575), "protocol": "TCP", "advertise": false}, @@ -246,6 +255,11 @@ func isolationTemplate(t *testing.T, name string, isolated bool) { func isolationGameIP(t *testing.T, ns, name string, isolated bool) string { t.Helper() + namespace, err := envInstance.K8s.CoreV1().Namespaces().Get(t.Context(), ns, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + restricted := namespace.Labels["pod-security.kubernetes.io/enforce"] == "restricted" var ip string envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name+"-0", metav1.GetOptions{}) @@ -255,6 +269,9 @@ func isolationGameIP(t *testing.T, ns, name string, isolated bool) string { if pod.DeletionTimestamp != nil || (pod.Labels["gameplane.local/sentinel-isolation"] == "true") != isolated { return false, "waiting for rollout with expected isolation label" } + if restricted && pod.Annotations["gameplane.local/workload-security"] != "restricted" { + return false, "waiting for replacement game pod after the restricted profile rollout" + } for _, condition := range pod.Status.Conditions { if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { ip = pod.Status.PodIP @@ -281,7 +298,16 @@ func isolationClient(t *testing.T, ns, name string, labels map[string]string) st noToken := false pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, - Containers: []corev1.Container{{Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}}}}, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptr.To(true), RunAsUser: ptr.To(int64(1000)), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Containers: []corev1.Container{{ + Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + }, + }}, + }, } if _, err := envInstance.K8s.CoreV1().Pods(ns).Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { t.Fatal(err) @@ -303,12 +329,17 @@ func isolationHTTP(t *testing.T, ns, source, ip string, port int, allowed bool) t.Helper() // Every exec must succeed regardless of network outcome. Otherwise a broken // kubectl/CNI fixture could masquerade as the expected denied connection. - target := fmt.Sprintf("http://%s:%d/", ip, port) + isolationAttempt(t, ns, source, allowed, `wget -q -T 2 -O /dev/null "$1"`, fmt.Sprintf("http://%s:%d/", ip, port)) +} + +// Each command is fixed by the test. Pass hosts, ports and URLs separately. +func isolationAttempt(t *testing.T, ns, source string, allowed bool, command string, args ...string) { + t.Helper() envInstance.Eventually(t, 45*time.Second, func() (bool, string) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() - out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, source, "--", - "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) + execArgs := []string{"exec", "-n", ns, source, "--", "sh", "-c", "if " + command + "; then echo ALLOWED; else echo DENIED; fi", "probe"} + out, err := envInstance.Kubectl(ctx, append(execArgs, args...)...) if err != nil { return false, "probe execution failed: " + err.Error() + out } @@ -317,7 +348,7 @@ func isolationHTTP(t *testing.T, ns, source, ip string, port int, allowed bool) if allowed { want = "ALLOWED" } - return got == want, fmt.Sprintf("%s -> %s: got %q, want %s; output: %q", source, target, got, want, out) + return got == want, fmt.Sprintf("%s -> %v: got %q, want %s; output: %q", source, args, got, want, out) }) } diff --git a/test/e2e/network_peers_e2e_test.go b/test/e2e/network_peers_e2e_test.go new file mode 100644 index 000000000..ce92045dd --- /dev/null +++ b/test/e2e/network_peers_e2e_test.go @@ -0,0 +1,167 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +func checkTenantEndpoints(t *testing.T, ns, tenant, server, localPeer, homeIP, awayIP, peerIP string, profileApplied time.Time) { + t.Helper() + monitor := isolationClient(t, "monitoring", server+"-monitor", nil) + const fetchPeer = `wget --no-check-certificate -q -T 3 -O - "$1" | grep -Fxq isolated-kind-peer` + const scrape = `wget -q -T 3 -O - "$1" | grep -q '^gameplane_agent_'` + const connect = `nc -w 3 "$1" "$2" %s: %v %s", url, err, out) + }) +} + +func enableIsolatedFRP(t *testing.T, ns, server string) { + t.Helper() + ctx := t.Context() + envInstance.ApplyYAML(t, "frps.yaml") + game, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, server, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + auth := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: server + "-frp", OwnerReferences: []metav1.OwnerReference{ + {APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: server, UID: game.GetUID()}, + }}, StringData: map[string]string{"token": ocFrpsToken}} + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Create(ctx, auth, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + // This is the administrator-supplied provider exception documented for + // tenant isolation. Cilium evaluates the selected frps backend after DNAT. + port := intstr.FromInt32(7000) + policy := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: server + "-frp-provider"}, + Spec: networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": "gameplane-tunnel", "app.kubernetes.io/instance": server}}, + PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeEgress}, + Egress: []networkingv1.NetworkPolicyEgressRule{{To: []networkingv1.NetworkPolicyPeer{{ + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ocNS}}, + PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app.kubernetes.io/name": "e2e-frps"}}, + }}, Ports: []networkingv1.NetworkPolicyPort{{Port: &port, Protocol: ptr.To(corev1.ProtocolTCP)}}}}, + }} + if _, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ns).Create(ctx, policy, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.NetworkingV1().NetworkPolicies(ns).Delete(context.Background(), policy.Name, metav1.DeleteOptions{}) + }) + patch := []byte(fmt.Sprintf(`{"spec":{"networking":{"tunnel":{"enabled":true,"provider":"frp","credentialsSecretRef":{"name":%q},"frp":{"serverAddr":%q,"remotePorts":[{"name":"game","remotePort":30000}]}}}}}`, auth.Name, ocFrpsAddr)) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, server, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + out, err := envInstance.Kubectl(ctx, "rollout", "status", "-n", ocNS, "deployment/e2e-frps", "--timeout=2m") + if err != nil { + t.Fatalf("frps: %v %s", err, out) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, server, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return ocTunnelReady(got) + }) + peers, err := envInstance.K8s.CoreV1().Pods(ocNS).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=e2e-frps"}) + if err != nil || len(peers.Items) != 1 { + t.Fatalf("frps pods: %v %v", peers, err) + } + control := isolationClient(t, "monitoring", server+"-frp-player", nil) + isolationAttempt(t, "monitoring", control, true, `wget -q -T 3 -O - "$1" | grep -Fxq reachable`, "http://"+peers.Items[0].Status.PodIP+":30000/") +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 047e00451..0f300c26e 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -11,9 +11,15 @@ The amd64 `network-isolation` bucket requires the Cilium fixture cluster. It tests actual legacy versus isolated sentinel connections, an existing template's opt-in rollout without selector changes, HTTP kubelet readiness, fresh agent heartbeats, and API-to-agent mTLS access. The tenant -profile drill exercises same-named games in separate namespaces, relay-to-game -and relay-to-waker access, revocation, and per-game versus shared probe ports. It uses one -admin login and runs separately from the kindnet shape-only checks. +profile drill exercises same-named games in two Restricted namespaces, real +frp forwarding, relay-to-waker access and revocation, and per-game versus +shared probe ports. A player on the kind bridge connects through a NodePort +with its source address preserved. An operator-created Backup writes to a Restic REST +repository on the bridge. A monitoring client reads both agents' metrics. +The drill checks fresh heartbeats after policy rollout and denies lateral agent +and RCON connections, private HTTPS and a reachable link-local metadata fixture. +`deploy/kind/with-network-peers.sh` installs the fixture route only inside the +disposable node and removes it and its peer containers on exit. ### Restricted workload drill (#714) diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 803e2917f..41696fa42 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -20,7 +20,8 @@ { "tests": "^(TestPanel_OfflineRecovery|TestAPIClient_DoContextResponseBody|TestAPIClient_ExtractIntField)$", "components": ["operator", "api", "agent"], "paths": ["test/e2e/testdata/recovery/"] }, - { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent", "tunnel"], + "paths": ["test/e2e/fixtures/frps.yaml"] }, { "tests": "^TestMonitoring_DiscoveryPerformanceAndPolicy$", "components": ["operator", "api", "agent", "tunnel"] }, { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, From ab32b80c223e87830cc70c9e5983f33f6cbb4e32 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 10 Nov 2026 23:33:35 -0500 Subject: [PATCH 4/6] test(networking): parse fixture endpoints before invoking clients Signed-off-by: Baylem --- test/e2e/network_isolation_e2e_test.go | 7 +------ test/e2e/network_peers_e2e_test.go | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 1a74b309c..bf2c5e928 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -5,8 +5,6 @@ package e2e import ( "context" "fmt" - "net" - "os" "reflect" "strings" "testing" @@ -114,10 +112,7 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { t.Cleanup(func() { _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) }) - peerIP := os.Getenv("GAMEPLANE_NET_PEER_IP") - if ip := net.ParseIP(peerIP); ip == nil || ip.To4() == nil || !ip.IsPrivate() { - t.Fatal("network isolation requires the private IPv4 endpoint from deploy/kind/with-network-peers.sh") - } + peerIP := kindFixtureIP(t, "GAMEPLANE_NET_PEER_IP") helmUpgradeReuse(ctx, t, "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", "podSecurity.enforceRestricted=true", "networkPolicies.probePorts=null", "networkPolicies.gameEgress.enabled=false", "networkPolicies.backupEgress.enabled=true", "networkPolicies.backupEgress.toCIDRs[0]="+peerIP+"/32", diff --git a/test/e2e/network_peers_e2e_test.go b/test/e2e/network_peers_e2e_test.go index ce92045dd..f36ef09b6 100644 --- a/test/e2e/network_peers_e2e_test.go +++ b/test/e2e/network_peers_e2e_test.go @@ -4,8 +4,9 @@ package e2e import ( "context" + "encoding/hex" "fmt" - "net" + "net/netip" "os" "os/exec" "strings" @@ -21,6 +22,17 @@ import ( "k8s.io/utils/ptr" ) +// The wrapper discovers both endpoints on kind's private IPv4 bridge. Render +// the parsed address before passing it to Helm, kubectl or Docker. +func kindFixtureIP(t *testing.T, key string) string { + t.Helper() + ip, err := netip.ParseAddr(os.Getenv(key)) + if err != nil || !ip.Is4() || !ip.IsPrivate() { + t.Fatalf("%s must identify a private IPv4 address on the kind bridge", key) + } + return ip.String() +} + func checkTenantEndpoints(t *testing.T, ns, tenant, server, localPeer, homeIP, awayIP, peerIP string, profileApplied time.Time) { t.Helper() monitor := isolationClient(t, "monitoring", server+"-monitor", nil) @@ -100,10 +112,12 @@ func checkExternalPlayer(t *testing.T, ns, server string) { t.Cleanup(func() { _ = envInstance.K8s.CoreV1().Services(ns).Delete(context.Background(), svc.Name, metav1.DeleteOptions{}) }) - node, client := os.Getenv("GAMEPLANE_NET_NODE_IP"), os.Getenv("GAMEPLANE_NET_PEER_CONTAINER") - if net.ParseIP(node) == nil || client == "" { - t.Fatal("missing kind network peer or node") + node := kindFixtureIP(t, "GAMEPLANE_NET_NODE_IP") + id, err := hex.DecodeString(os.Getenv("GAMEPLANE_NET_PEER_CONTAINER")) + if err != nil || len(id) != 32 { + t.Fatal("expected the full container ID from the kind network fixture") } + client := hex.EncodeToString(id) url := fmt.Sprintf("http://%s:%d/", node, svc.Spec.Ports[0].NodePort) envInstance.Eventually(t, time.Minute, func() (bool, string) { out, err := exec.CommandContext(t.Context(), "docker", "exec", client, "wget", "-T", "3", "-qO-", url).CombinedOutput() From 1d99e02a820887360bae4acbd5ba2d72e10866b6 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 00:58:42 -0500 Subject: [PATCH 5/6] fix(networking): clear persisted shared probe grants on tenant isolation upgrade Signed-off-by: Baylem --- docs/security.md | 4 ++++ test/e2e/network_isolation_e2e_test.go | 11 ++++++++--- test/e2e/telemetry_e2e_helpers_test.go | 12 +++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/security.md b/docs/security.md index e811d95b5..51619240a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -349,6 +349,10 @@ destinations require a provider-aware egress policy or a separately isolated deployment. Check existing provider policies after the operator reconciles. Run positive joins, heartbeat, scrape and backup checks on the deployed CNI. +When upgrading with Helm's `--reuse-values` from a shared kubelet probe list, +pass `--set-json 'networkPolicies.probePorts=[]'` to replace the previous +list explicitly. Check that each managed game namespace has its per-server +probe policies and that the legacy `allow-kubelet-probes` grant is absent. From an independent tenant, check the target Pod IP and Service for RCON and agent access. Verify private HTTPS and metadata endpoints are unreachable. Review additional policies in each namespace because their grants are additive. diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index bf2c5e928..67aab2a18 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -113,8 +113,9 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { _ = envInstance.K8s.CoreV1().Namespaces().Delete(context.Background(), tenant, metav1.DeleteOptions{}) }) peerIP := kindFixtureIP(t, "GAMEPLANE_NET_PEER_IP") - helmUpgradeReuse(ctx, t, "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", - "podSecurity.enforceRestricted=true", "networkPolicies.probePorts=null", "networkPolicies.gameEgress.enabled=false", + helmUpgradeReuseJSON(ctx, t, []string{"networkPolicies.probePorts=[]"}, + "operator.gameNamespaces[0]="+tenant, "networkPolicies.tenantIsolation=true", + "podSecurity.enforceRestricted=true", "networkPolicies.gameEgress.enabled=false", "networkPolicies.backupEgress.enabled=true", "networkPolicies.backupEgress.toCIDRs[0]="+peerIP+"/32", "networkPolicies.backupEgress.ports[0].port=8000", "networkPolicies.backupEgress.ports[0].protocol=TCP", "networkPolicies.gameIngress.fromCIDRs[0]="+peerIP+"/32") @@ -142,7 +143,11 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { isolationHTTP(t, ns, probeA, away, 25566, false) isolationHTTP(t, tenant, otherProbe, home, 25566, false) for namespace, want := range map[string]int{ns: 25566, tenant: 25575} { - policy, err := envInstance.K8s.NetworkingV1().NetworkPolicies(namespace).Get(ctx, a+"-probes", metav1.GetOptions{}) + policies := envInstance.K8s.NetworkingV1().NetworkPolicies(namespace) + if _, err := policies.Get(ctx, "allow-kubelet-probes", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("shared kubelet grant survived tenant migration in %s: %v", namespace, err) + } + policy, err := policies.Get(ctx, a+"-probes", metav1.GetOptions{}) if err != nil || len(policy.Spec.Ingress) != 1 || len(policy.Spec.Ingress[0].Ports) != 1 || policy.Spec.Ingress[0].Ports[0].Port == nil || policy.Spec.Ingress[0].Ports[0].Port.IntValue() != want { t.Fatalf("expected only kubelet probe port %d in %s: %v", want, namespace, err) diff --git a/test/e2e/telemetry_e2e_helpers_test.go b/test/e2e/telemetry_e2e_helpers_test.go index cb488897a..3f52d3c68 100644 --- a/test/e2e/telemetry_e2e_helpers_test.go +++ b/test/e2e/telemetry_e2e_helpers_test.go @@ -252,15 +252,25 @@ func tlDo(ctx context.Context, c *APIClient, method, path string, body any) (int // helmUpgradeReuse runs `helm upgrade --reuse-values` with the given --set // values on the e2e release and waits for the rollout. func helmUpgradeReuse(ctx context.Context, t *testing.T, sets ...string) { + t.Helper() + helmUpgradeReuseJSON(ctx, t, nil, sets...) +} + +// Helm merges persisted release values on upgrade. Use --set-json when the +// replacement must remain an explicitly empty array through that merge. +func helmUpgradeReuseJSON(ctx context.Context, t *testing.T, jsonSets []string, sets ...string) { t.Helper() args := []string{"upgrade", "gameplane", filepath.Join("..", "..", "charts", "gameplane"), "--kube-context", envInstance.Context, "--namespace", "gameplane-system", "--reuse-values"} for _, set := range sets { args = append(args, "--set", set) } + for _, set := range jsonSets { + args = append(args, "--set-json", set) + } args = append(args, "--wait", "--timeout", "6m") if out, err := exec.CommandContext(ctx, "helm", args...).CombinedOutput(); err != nil { - t.Fatalf("helm upgrade %v failed: %v\n%s", sets, err, out) + t.Fatalf("helm upgrade %v (JSON values %v) failed: %v\n%s", sets, jsonSets, err, out) } } From e40591583a57f2ea945161fed134557e28c7ff73 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 11 Nov 2026 03:36:53 -0500 Subject: [PATCH 6/6] test(networking): resolve named kubelet grants against the target pod Signed-off-by: Baylem --- test/e2e/network_isolation_e2e_test.go | 41 +++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 67aab2a18..12c05206e 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -15,6 +15,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" ) @@ -142,15 +143,47 @@ func checkTenantNetworkProfile(t *testing.T, ns, a, b, probeA string) { isolationHTTP(t, tenant, otherProbe, away, 25566, true) isolationHTTP(t, ns, probeA, away, 25566, false) isolationHTTP(t, tenant, otherProbe, home, 25566, false) - for namespace, want := range map[string]int{ns: 25566, tenant: 25575} { + for namespace, want := range map[string]int32{ns: 25566, tenant: 25575} { policies := envInstance.K8s.NetworkingV1().NetworkPolicies(namespace) if _, err := policies.Get(ctx, "allow-kubelet-probes", metav1.GetOptions{}); !apierrors.IsNotFound(err) { t.Fatalf("shared kubelet grant survived tenant migration in %s: %v", namespace, err) } policy, err := policies.Get(ctx, a+"-probes", metav1.GetOptions{}) - if err != nil || len(policy.Spec.Ingress) != 1 || len(policy.Spec.Ingress[0].Ports) != 1 || - policy.Spec.Ingress[0].Ports[0].Port == nil || policy.Spec.Ingress[0].Ports[0].Port.IntValue() != want { - t.Fatalf("expected only kubelet probe port %d in %s: %v", want, namespace, err) + if err != nil { + t.Fatal(err) + } + if len(policy.Spec.Ingress) != 1 || len(policy.Spec.Ingress[0].Ports) == 0 { + t.Fatalf("missing restricted kubelet rule for %s/%s: %+v", namespace, a, policy.Spec) + } + rule := policy.Spec.Ingress[0] + if len(rule.From) != 1 || rule.From[0].IPBlock == nil || rule.From[0].IPBlock.CIDR != kindFixtureIP(t, "GAMEPLANE_NET_NODE_IP")+"/32" { + t.Fatalf("unexpected kubelet sources for %s/%s: %+v", namespace, a, rule.From) + } + pod, err := envInstance.K8s.CoreV1().Pods(namespace).Get(ctx, a+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + // Startup names the socket while readiness uses its number. Kubernetes + // resolves a named NetworkPolicy port against the destination Pod. + namedTCP := map[string]int32{} + for _, c := range pod.Spec.Containers { + for _, port := range c.Ports { + if port.Protocol == corev1.ProtocolTCP && port.Name != "" { + namedTCP[port.Name] = port.ContainerPort + } + } + } + for _, port := range rule.Ports { + if port.Port == nil || port.EndPort != nil || port.Protocol == nil || *port.Protocol != corev1.ProtocolTCP { + t.Fatalf("unexpected kubelet grant for %s/%s: %+v", namespace, a, port) + } + effective := port.Port.IntVal + if port.Port.Type == intstr.String { + effective = namedTCP[port.Port.StrVal] + } + if effective != want { + t.Fatalf("kubelet port %v resolves to %d in %s/%s, want only TCP %d", port.Port, effective, namespace, a, want) + } } }