diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48413c374..a9cbc571e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1257,6 +1257,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: strict network profile and sentinel policy selectors + run: python3 charts/tests/check_network_isolation.py - name: workload security chart contract run: python3 charts/tests/check_workload_security.py - name: auxiliary resources and management placement chart renders @@ -1824,9 +1826,13 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement, backup-preparation] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, network-isolation, management-placement, backup-preparation] arch: [amd64] include: + - bucket: network-isolation + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: operator parallel: 8 test_timeout: 20m @@ -1900,6 +1906,7 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: + GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }} GAMEPLANE_E2E_PROFILE: ${{ (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) @@ -2728,7 +2735,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|network-isolation|management-placement|backup-preparation) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2881,7 +2888,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|network-isolation|management-placement|backup-preparation) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d62bfcc61..6dfd64e5b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 91db42032..50bf98570 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 7d5cca499..21be24ac3 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index aaa82dabb..c07187666 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index c039b349b..a19fd9526 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 9de0b6e6f..5b725e7ff 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index bfc2cd543..9739185df 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index e3c4e16f4..5d6672bd3 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index 13f62e976..32d1ebfcb 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d62bfcc61..6dfd64e5b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 91db42032..50bf98570 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 7d5cca499..21be24ac3 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index aaa82dabb..c07187666 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index c039b349b..a19fd9526 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 9de0b6e6f..5b725e7ff 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index bfc2cd543..9739185df 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index e3c4e16f4..5d6672bd3 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index 13f62e976..32d1ebfcb 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 + gameplane.local/crd-bundle-sha256: 98e3825f4571f50622830dbb04e97fdb3e45f8be8bc679259dd17c5d30eec884 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index 81f963f4d..612fc5071 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -331,8 +331,9 @@ spec: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } --- -# Allow the sentinel to egress to game pods on advertised ports for the proxy path. -# This is the INGRESS half only, on the game pod's own selector. +# Legacy sentinel proxy allowance, ingress half. This grants ALL ports between +# non-isolated sentinels and games. NetworkPolicies are additive; per-server +# player ingress does not narrow it. Opted-in templates use operator policies. # NetworkPolicy requires both sides to agree: the destination (this policy) # admits the sentinel, but default-deny-egress's podSelector: {} still # governs the sentinel as a SOURCE and only opens DNS. allow-sentinel-to-game-egress @@ -349,14 +350,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-game + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Ingress] ingress: - from: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker - # All ports; the per-server ingress policy covers which ports are - # actually advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. --- # The Egress half of allow-sentinel-to-game (see its comment above): admits # the sentinel, as a traffic SOURCE, to reach game pods. Selects the @@ -374,15 +378,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Egress] egress: - to: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-game - # All ports; allow-sentinel-to-game above (the matching ingress half) - # and the per-server ingress policy cover which ports are actually - # advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index e70bc87dc..ec05eac8f 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,31 @@ +{{- $network := .Values.networkPolicies | default dict }} +{{- if $network.strict }} +{{- if not $network.enabled }} +{{- fail "networkPolicies.strict requires networkPolicies.enabled=true" }} +{{- end }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" "probePorts" }} +{{- $entries := index $network $key }} +{{- if or (not (kindIs "slice" $entries)) (eq (len $entries) 0) }} +{{- fail (printf "networkPolicies.strict requires a non-empty networkPolicies.%s list" $key) }} +{{- end }} +{{- end }} +{{- $octet := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])" }} +{{- $ipv4 := printf "^%s(\\.%s){3}/([0-9]|[12][0-9]|3[0-2])$" $octet $octet }} +{{- $h := "[0-9A-Fa-f]{1,4}" }} +{{- $ipv6 := printf "^((%s:){7}%s|(%s:){1,7}:|(%s:){1,6}:%s|(%s:){1,5}(:%s){1,2}|(%s:){1,4}(:%s){1,3}|(%s:){1,3}(:%s){1,4}|(%s:){1,2}(:%s){1,5}|%s:((:%s){1,6})|:((:%s){1,7}|:))/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" }} +{{- range (index $network $key) }} +{{- if or (not (kindIs "string" .)) (not (or (regexMatch $ipv4 (toString .)) (regexMatch $ipv6 (toString .)))) }} +{{- fail (printf "networkPolicies.%s entries must be explicit IPv4/IPv6 CIDRs" $key) }} +{{- end }} +{{- end }} +{{- end }} +{{- range $network.probePorts }} +{{- if or (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} +{{- fail "networkPolicies.probePorts requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" }} +{{- end }} +{{- end }} +{{- end }} {{- $diagnostics := include "gameplane.nodeDiagnostics" . | fromJson }} {{- if $diagnostics.enabled }} {{- if or (not .Values.capture.enabled) (ne (include "gameplane.operatorEnabled" .) "true") }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index d660889ac..9c933671a 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -564,6 +564,9 @@ gamesNamespace: gameplane-games networkPolicies: enabled: true + # Fail closed unless actual kubelet source CIDRs/probe ports and apiserver + # endpoint CIDRs are supplied. Defaults retain the legacy broad allowances. + strict: false # CIDRs that hold kubelet source IPs for liveness/readiness probes. # If unset, the chart falls back to RFC1918 + link-local ranges, # which cover most clusters without opening the public internet. diff --git a/charts/tests/check_network_isolation.py b/charts/tests/check_network_isolation.py new file mode 100644 index 000000000..7c180d143 --- /dev/null +++ b/charts/tests/check_network_isolation.py @@ -0,0 +1,49 @@ +"""CI-only rendered effective policy tuple and strict-profile regressions.""" +from check_standalone import render, one + + +def main(): + values = {"networkPolicies": {"strict": True, + "kubeletCIDRs": ["192.0.2.10/32"], + "probePorts": [{"protocol": "TCP", "port": 8090}], + "apiServerCIDRs": ["198.51.100.20/32"]}} + objects = render(values) + probe = one(objects, "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"] + assert probe == [{"from": [{"ipBlock": {"cidr": "192.0.2.10/32"}}], + "ports": [{"protocol": "TCP", "port": 8090}]}] + for name in ("allow-agent-to-apiserver", "allow-sentinel-to-apiserver"): + egress = one(objects, "NetworkPolicy", name)["spec"]["egress"] + assert egress == [{"to": [{"ipBlock": {"cidr": "198.51.100.20/32"}}], + "ports": [{"protocol": "TCP", "port": 443}, {"protocol": "TCP", "port": 6443}]}] + excluded = [{"key": "gameplane.local/sentinel-isolation", "operator": "NotIn", "values": ["true"]}] + for name, direction, peer in (("allow-sentinel-to-game", "ingress", "from"), + ("allow-sentinel-to-game-egress", "egress", "to")): + policy = one(objects, "NetworkPolicy", name)["spec"] + assert policy["podSelector"]["matchExpressions"] == excluded + assert policy[direction][0][peer][0]["podSelector"]["matchExpressions"] == excluded + import copy + for key in ("kubeletCIDRs", "apiServerCIDRs", "probePorts"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"][key] = [] + render(invalid, failure="requires a non-empty networkPolicies." + key) + for cidr in ("", "10.0.0.1", "999.0.0.1/32", "10.0.0.1/33", "2001:::1/64", "2001:db8/64", "::/129"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(invalid, failure="must be explicit IPv4/IPv6 CIDRs") + for cidr in ("::/0", "::1/128", "2001:db8::/64", "2001:db8:1:2:3:4:5:6/128"): + valid = copy.deepcopy(values) + valid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(valid) + for port in (0, 65536, 1.5, "all"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["probePorts"][0]["port"] = port + render(invalid, failure="integer port") + render({"networkPolicies": {"strict": True, "enabled": False}}, failure="requires networkPolicies.enabled=true") + # Legacy/default installs still render the documented broad fallback. + fallback = one(render(), "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"][0] + assert "ports" not in fallback and {p["ipBlock"]["cidr"] for p in fallback["from"]} == { + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16"} + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 8f5df5211..3cf78410c 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -27,6 +27,11 @@ ACTION="${1:-up}" CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" +CNI="${GAMEPLANE_E2E_CNI:-kindnet}" +case "${CNI}" in +kindnet | cilium) ;; +*) echo "unsupported E2E CNI: ${CNI}" >&2; exit 2 ;; +esac case "${PROFILE}" in default | postgres | management) ;; *) @@ -188,14 +193,21 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (${PROFILE})" + echo "creating kind cluster ${CLUSTER} (${PROFILE}, ${CNI})" + kind_networking="" + kind_wait=90s + if [ "${CNI}" = cilium ]; then + kind_networking=$'networking:\n disableDefaultCNI: true' + kind_wait=0s # Nodes become Ready after Cilium is installed below. + fi extra_nodes="" if [[ "${PROFILE}" == "management" ]]; then extra_nodes=$' - role: worker\n - role: worker' fi - kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null + if [ "${CNI}" = cilium ]; then + # Pin the enforcing CNI; kindnet does not implement NetworkPolicy. + # https://docs.cilium.io/en/stable/installation/kind/ + helm upgrade --install cilium cilium --repo https://helm.cilium.io/ \ + --version 1.20.3 --namespace kube-system \ + --set image.pullPolicy=IfNotPresent --set ipam.mode=kubernetes \ + --set policyCIDRMatchMode=nodes \ + --set operator.replicas=1 --wait --timeout 5m + kubectl wait nodes --all --for=condition=Ready --timeout=120s + fi + install_metallb apply_metallb_pools @@ -363,6 +386,17 @@ EOF deployment/gameplane-test-fakeoidc --timeout=120s DB_SETS=() + NETWORK_SETS=() + if [ "${CNI}" = cilium ]; then + # Single-node IPv4 fixture: use actual endpoint/source addresses. + node_ip="$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" + api_ip="$(kubectl get endpoints kubernetes -n default -o jsonpath='{.subsets[0].addresses[0].ip}')" + NETWORK_SETS=(--set networkPolicies.strict=true + --set-string "networkPolicies.kubeletCIDRs[0]=${node_ip}/32" + --set-string "networkPolicies.apiServerCIDRs[0]=${api_ip}/32" + --set networkPolicies.probePorts[0].protocol=TCP + --set networkPolicies.probePorts[0].port=25566) + fi if [ "${PROFILE}" = postgres ]; then dsn="$(kubectl get secret gameplane-postgres --namespace gameplane-system \ -o jsonpath='{.data.dsn}' | base64 -d)" @@ -408,6 +442,7 @@ EOF --set "defaultModuleSource.enabled=false" \ --set "api.telemetry.receiver.enabled=true" \ ${DB_SETS[@]+"${DB_SETS[@]}"} \ + ${NETWORK_SETS[@]+"${NETWORK_SETS[@]}"} \ --wait --timeout 5m echo diff --git a/docs/module-authoring.md b/docs/module-authoring.md index a0553c207..3fbebaa9c 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -12,6 +12,15 @@ materializes into an in-cluster `GameTemplate`. ## Quickstart: Authoring with `gp-module` CLI & Web Builder +For a validated wake proxy, add `sentinelIsolation: true` under the template's +`spec`. The operator permits that server's sentinel to reach only its own +advertised container ports, preserving each port's TCP/UDP/SCTP protocol. +Leave management ports unadvertised. A template with no advertised ports grants +no sentinel access. Omission preserves legacy behavior; adoption in the +GameplanePanel/module repository is separate from the operator upgrade. +Test player joins, status queries and sleep/wake cycles with an enforcing CNI +before publishing the opt-in. See [network policy migration](networking.md#networkpolicy-isolation-and-migration). + Gameplane provides a unified toolkit for creating, verifying, previewing, and packaging modules: - **Command-Line Interface (`gp-module`)**: First-class Go CLI with Makefile targets. diff --git a/docs/networking.md b/docs/networking.md index a8415fb72..fd3259012 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -2,6 +2,66 @@ Gameplane exposes game servers through Kubernetes Services, with optional integration with your cluster's load-balancer address manager to assign specific public addresses or address pools to each server. +## NetworkPolicy isolation and migration + +NetworkPolicies are additive. An allowance in one policy cannot be narrowed by +another policy. Enforcement also requires a CNI that implements NetworkPolicy. + +Modules may set `GameTemplate.spec.sentinelIsolation: true`. The operator then +labels the game and sentinel pod templates and creates two owned policies: +`-sentinel-ingress` and `-sentinel-egress`. They allow a sentinel +to reach only its own server's advertised container ports, with the declared +protocol. Zero advertised ports creates no allowance. Management ports must +remain `advertise: false`. The legacy chart policies exclude opted-in sources +and destinations. Templates without the field retain the existing broad +sentinel-to-game allowance; module maintainers can adopt it after validating +their proxy traffic. + +Upgrade the CRDs, operator and chart together before opting in a template. +Existing game pods roll out to acquire the isolation label; StatefulSet +selectors stay unchanged. Do not claim isolation until both game and sentinel +pods have the label and the per-server policies exist. Other administrator-added +policies can still grant wider access. Disabling chart NetworkPolicies or using +a non-enforcing CNI does not provide this boundary. + +Probe and API-server narrowing is a separate installation option: + +```yaml +networkPolicies: + enabled: true + strict: true + kubeletCIDRs: ["192.0.2.10/32"] # Replace with observed kubelet sources. + probePorts: + - {protocol: TCP, port: 25565} # Replace with all configured probe ports. + apiServerCIDRs: ["198.51.100.20/32"] # Replace with actual API endpoints. +``` + +Strict mode rejects missing/empty lists, malformed CIDRs and invalid probe +ports. It never supplies broad fallback ranges. API egress is limited to the +listed addresses on TCP 443/6443. Choose addresses based on your CNI's policy +evaluation point: service DNAT or source SNAT can change which address it sees. +Include every control-plane endpoint and kubelet source that can host a server; +update these lists when nodes or endpoints change. CNI host-traffic exceptions +may still apply. + +On Cilium, CIDR rules exclude node identities by default. For a self-hosted +API server on a node, configure `policyCIDRMatchMode=nodes` before using these +plain NetworkPolicy endpoint CIDRs, or provide equivalent Cilium-specific +API-server policy. The CI fixture uses nodes mode. See +[Cilium CIDR policy semantics](https://docs.cilium.io/en/stable/security/policy/layer3/#selecting-pods-or-nodes-with-cidr-ipblock) +and validate kubelet source handling for your Cilium configuration as well. + +Before enabling strict mode, inventory the actual endpoints and configured +game probes, observe their traffic in the CNI, then validate game readiness, +agent heartbeats, wake requests and API/operator agent access in a staging +namespace. The default profile retains RFC1918/link-local API egress and +unrestricted probe ports for compatibility; it does not isolate those paths. +Narrowing sentinel traffic alone cannot remove those separate grants. + +The dedicated CI `network-isolation` bucket installs pinned Cilium, exercises +real allowed/denied connections, and upgrades existing legacy templates. +The default kindnet fixture only verifies policy shape. + ## Address Pool Overview When a GameServer is exposed as a LoadBalancer (`spec.networking.expose: LoadBalancer`), it can request a specific external address or address pool. The operator translates this request to your cluster's load-balancer flavor so the same CRD works whether you run MetalLB, Cilium, or no address manager at all. diff --git a/hack/ci_scope.py b/hack/ci_scope.py index 492efd102..f6fc57387 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,8 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation"] -E2E_AMD64_ONLY = {"management-placement", "backup-preparation"} + "telemetry", "optional-components", "restricted-workloads", "network-isolation", "management-placement", "backup-preparation"] +E2E_AMD64_ONLY = {"network-isolation", "management-placement", "backup-preparation"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index b5097a3ec..33bcfeba1 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -408,6 +408,17 @@ def select(self, *paths): """Return the e2e outputs for a partial-scope change to paths.""" return e2e_outputs(ROOT, list(paths), None) + def test_audit_network_isolation_has_a_run_pattern(self): + """Affected components and the owning test file select the audit bucket.""" + for path in ("operator/internal/controller/gameserver_controller.go", + "api/cmd/main.go", + "agent/cmd/main.go", + "test/e2e/network_isolation_e2e_test.go"): + with self.subTest(path=path): + values = self.select(path) + self.assertIn("TestNetworkIsolation_EnforcedAndUpgrade", + values["e2e-run"].get("network-isolation", "")) + def test_audit_restricted_workloads_has_a_run_pattern(self): """The node capture worker and all drill components retain CI coverage.""" for path in ("operator/internal/controller/workload_security.go", diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index 8001a2b27..873fbf31b 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -89,6 +89,12 @@ type GameTemplateSpec struct { // +optional Ports []GamePort `json:"ports,omitempty"` + // SentinelIsolation restricts a wake sentinel to its own server's advertised + // ports. Templates opt in after module connectivity validation; absent or + // false retains the legacy namespace-wide sentinel allowances. + // +optional + SentinelIsolation bool `json:"sentinelIsolation,omitempty"` + // Storage describes the default persistent storage layout. // +optional Storage GameStorageSpec `json:"storage,omitempty"` diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 729b872e1..20f849800 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2113,6 +2113,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 1ef3a0148..c6c637ced 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -3,6 +3,7 @@ package controller import ( "context" "encoding/json" + "errors" "fmt" "net/netip" "path" @@ -287,6 +288,13 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } tmpl := *effective + if tmpl.Spec.SentinelIsolation && len(networkPolicyPortsFromTemplate(&tmpl)) == 0 { + cleanupErr := r.revokeInvalidSentinelIsolationExposure(ctx, &gs) + phaseErr := r.setPhase(ctx, &gs, + "sentinelIsolation requires at least one advertised port; advertise a game port or disable sentinelIsolation in the GameTemplate") + return ctrl.Result{}, errors.Join(cleanupErr, phaseErr) + } + // Resolve spec.config against the template's configSchema before // touching any children: invalid config must fail loudly (mirroring // the missing-template path) instead of materializing a pod that @@ -330,6 +338,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "reconcile ingress NetworkPolicy") return ctrl.Result{}, err } + if err := r.reconcileSentinelIsolation(ctx, &gs, &tmpl); err != nil { + return ctrl.Result{}, err + } if err := r.reconcileAgentTLS(ctx, &gs); err != nil { logger.Error(err, "reconcile agent TLS") return ctrl.Result{}, err @@ -1640,7 +1651,13 @@ func (r *GameServerReconciler) reconcileStatefulSet( } ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} - ss.Spec.Template.Labels = labels + ss.Spec.Template.Labels = make(map[string]string, len(labels)+1) + for key, value := range labels { + ss.Spec.Template.Labels[key] = value + } + if tmpl.Spec.SentinelIsolation { + ss.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Stamp (or clear) the config fingerprint without touching // annotations other actors may have set on the pod template. ann := ss.Spec.Template.Annotations diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index 5415fbbf8..c2549fc75 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -230,6 +230,9 @@ func (r *GameServerReconciler) reconcileSentinel( sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name, } + if tmpl.Spec.SentinelIsolation { + dep.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Container ports: advertised ports only, mirroring buildGameContainer. ports := make([]corev1.ContainerPort, 0) diff --git a/operator/internal/controller/gameserver_wipe_inventory_test.go b/operator/internal/controller/gameserver_wipe_inventory_test.go index 2ae64daf1..181cc9d57 100644 --- a/operator/internal/controller/gameserver_wipe_inventory_test.go +++ b/operator/internal/controller/gameserver_wipe_inventory_test.go @@ -13,6 +13,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/util/cert" ctrl "sigs.k8s.io/controller-runtime" @@ -29,6 +30,7 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { t.Fatal(err) } gs := wipeGameServer(false, "") + gs.Spec.TemplateRollout = &gameplanev1alpha1.TemplateRolloutSpec{Policy: "OnRequest"} tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: gs.Spec.TemplateRef.Name}, Spec: gameplanev1alpha1.GameTemplateSpec{Image: "example.invalid/game:unit"}} key, err := rsa.GenerateKey(rand.Reader, 2048) if err != nil { @@ -43,7 +45,7 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { "ca.key": pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), }} c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, tmpl, caSecret).WithStatusSubresource(gs).Build() - reader := &wipeInventoryReader{Reader: c} + reader := &wipeInventoryReader{Reader: c, sentinelNamespace: gs.Namespace, sentinelServer: gs.Name} r := &GameServerReconciler{Client: c, APIReader: reader, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} // This test measures an ordinary already-admitted reconcile. Template // admission has its own live worker checks and transition tests. @@ -63,8 +65,35 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { } // Exercise both initial child/status creation and the next full pass. Idle // and capture reconciliation participate, rather than just lifecycle helpers. - for range 2 { - reader.jobGets, reader.podLists = 0, 0 + for pass := range 2 { + if pass == 1 { + // An invalid unadmitted catalog revision must not revoke exposure + // or block rendering from the existing valid effective snapshot. + tmpl.Spec.SentinelIsolation = true + if err := c.Update(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + isolated := tmpl.DeepCopy() + isolated.Spec.SentinelIsolation = true + isolated.Spec.Ports = []gameplanev1alpha1.GamePort{{Name: "game", ContainerPort: 25565, Advertise: true}} + if err := r.reconcileSentinelIsolation(t.Context(), gs, isolated); err != nil { + t.Fatal(err) + } + // Measure a settled revision observation, as in the first pass. + // A status transition during reconcile changes ResourceVersion and + // correctly requires a fresh worker inventory before mutation. + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + _, desired, err := templateSnapshot(tmpl) + if err != nil { + t.Fatal(err) + } + if err := r.reportTemplateRevision(t.Context(), gs, desired, lifecycleOperation{}, false); err != nil { + t.Fatal(err) + } + } + reader.jobGets, reader.podLists, reader.filteredPodLists, reader.sentinelPodLists = 0, 0, 0, 0 if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { t.Fatal(err) } @@ -76,7 +105,10 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { t.Fatal("ordinary reconcile did not produce a running workload") } if reader.jobGets != 1 || reader.podLists != 1 { - t.Fatalf("full reconcile repeated its uncached worker inventory: Job GETs=%d Pod LISTs=%d", reader.jobGets, reader.podLists) + t.Fatalf("full reconcile pass %d repeated its uncached worker inventory: Job GETs=%d Pod LISTs=%d", pass, reader.jobGets, reader.podLists) + } + if reader.filteredPodLists != pass || reader.sentinelPodLists != pass { + t.Fatalf("unexpected filtered inventories: all=%d sentinel=%d want=%d", reader.filteredPodLists, reader.sentinelPodLists, pass) } } } @@ -110,7 +142,9 @@ func TestWipeInventoryCannotAuthorizeRecreatedWorkload(t *testing.T) { type wipeInventoryReader struct { client.Reader - jobGets, podLists int + jobGets, podLists int + filteredPodLists, sentinelPodLists int + sentinelNamespace, sentinelServer string } func (r *wipeInventoryReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { @@ -122,11 +156,48 @@ func (r *wipeInventoryReader) Get(ctx context.Context, key client.ObjectKey, obj func (r *wipeInventoryReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*corev1.PodList); ok { - r.podLists++ + options := (&client.ListOptions{}).ApplyOptions(opts) + // Wipe inventory deliberately reads every Pod in the namespace so + // unlabeled legacy workers remain visible. Filtered safety queries, + // such as sentinel opt-out drainage, are separate inventories. + if (options.LabelSelector == nil || options.LabelSelector.Empty()) && + (options.FieldSelector == nil || options.FieldSelector.Empty()) { + r.podLists++ + } else { + r.filteredPodLists++ + expected := labels.SelectorFromSet(labels.Set{"app.kubernetes.io/instance": r.sentinelServer, sentinelIsolationLabel: "true"}) + if r.sentinelServer != "" && options.Namespace == r.sentinelNamespace && options.LabelSelector != nil && options.LabelSelector.String() == expected.String() && (options.FieldSelector == nil || options.FieldSelector.Empty()) { + r.sentinelPodLists++ + } + } } return r.Reader.List(ctx, list, opts...) } +func TestWipeInventoryReaderSeparatesFilteredPodQueries(t *testing.T) { + reader := &wipeInventoryReader{Reader: fake.NewClientBuilder().WithScheme(wipeScheme(t)).Build(), sentinelNamespace: "games", sentinelServer: "server"} + for range 2 { + var pods corev1.PodList + if err := reader.List(t.Context(), &pods, client.InNamespace("games")); err != nil { + t.Fatal(err) + } + if err := reader.List(t.Context(), &pods, client.InNamespace("games"), client.MatchingLabels{ + "app.kubernetes.io/instance": "server", sentinelIsolationLabel: "true", + }); err != nil { + t.Fatal(err) + } + if err := reader.List(t.Context(), &pods, client.InNamespace("games"), client.MatchingLabels{wipeUIDLabel: "extra-wipe-query"}); err != nil { + t.Fatal(err) + } + } + if reader.podLists != 2 { + t.Fatalf("duplicate unfiltered inventories must count independently of sentinel queries: got %d", reader.podLists) + } + if reader.filteredPodLists != 4 || reader.sentinelPodLists != 2 { + t.Fatalf("filtered inventories were hidden or misclassified: all=%d sentinel=%d", reader.filteredPodLists, reader.sentinelPodLists) + } +} + func TestWipeInventoryIsSharedWithWorkloadMutation(t *testing.T) { s := wipeScheme(t) gs := wipeGameServer(false, "") @@ -140,8 +211,8 @@ func TestWipeInventoryIsSharedWithWorkloadMutation(t *testing.T) { if err := r.reconcileStatefulSet(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, 1, &workers); err != nil { t.Fatal(err) } - if reader.jobGets != 1 || reader.podLists != 1 { - t.Fatalf("ordinary reconcile inventoried workers repeatedly: Job GETs=%d Pod LISTs=%d", reader.jobGets, reader.podLists) + if reader.jobGets != 1 || reader.podLists != 1 || reader.filteredPodLists != 0 { + t.Fatalf("ordinary reconcile inventoried workers repeatedly: Job GETs=%d Pod LISTs=%d filtered=%d", reader.jobGets, reader.podLists, reader.filteredPodLists) } } @@ -184,8 +255,8 @@ func TestWipeInventoryRechecksChangedServerBeforeWorkloadMutation(t *testing.T) if guarded { wantLists = 1 } - if reader.podLists != wantLists { - t.Fatalf("Pod LISTs=%d want %d", reader.podLists, wantLists) + if reader.podLists != wantLists || reader.filteredPodLists != 0 { + t.Fatalf("Pod LISTs=%d filtered=%d want %d unfiltered", reader.podLists, reader.filteredPodLists, wantLists) } }) } @@ -209,7 +280,7 @@ func TestWipeReconcileReusesOwnedJobInventory(t *testing.T) { if err := r.reconcileWipe(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, &workers); err != nil { t.Fatal(err) } - if reader.jobGets != 1 || reader.podLists != 1 { - t.Fatalf("active wipe repeated its worker reads: Job GETs=%d Pod LISTs=%d", reader.jobGets, reader.podLists) + if reader.jobGets != 1 || reader.podLists != 1 || reader.filteredPodLists != 0 { + t.Fatalf("active wipe repeated its worker reads: Job GETs=%d Pod LISTs=%d filtered=%d", reader.jobGets, reader.podLists, reader.filteredPodLists) } } diff --git a/operator/internal/controller/gameserver_wipe_poll_test.go b/operator/internal/controller/gameserver_wipe_poll_test.go index 3f13a0abb..b3a347e07 100644 --- a/operator/internal/controller/gameserver_wipe_poll_test.go +++ b/operator/internal/controller/gameserver_wipe_poll_test.go @@ -105,8 +105,8 @@ func TestWipeFailurePollSharesInventoryWithLifecycle(t *testing.T) { if err := r.reconcileWipe(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, &workers); err != nil { t.Fatal(err) } - if reader.jobGets != 1 || reader.podLists != 1 { - t.Fatalf("failure polling repeated worker inventory: Job GETs=%d Pod LISTs=%d", reader.jobGets, reader.podLists) + if reader.jobGets != 1 || reader.podLists != 1 || reader.filteredPodLists != 0 { + t.Fatalf("failure polling repeated worker inventory: Job GETs=%d Pod LISTs=%d filtered=%d", reader.jobGets, reader.podLists, reader.filteredPodLists) } } diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go new file mode 100644 index 000000000..20a1057a3 --- /dev/null +++ b/operator/internal/controller/sentinel_isolation.go @@ -0,0 +1,156 @@ +package controller + +import ( + "context" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const sentinelIsolationLabel = "gameplane.local/sentinel-isolation" + +// The chart's legacy policies exclude opted-in pods on both sides. Policies +// are additive: these per-server grants cannot subtract a namespace-wide grant. +func (r *GameServerReconciler) reconcileSentinelIsolation(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) error { + if !tmpl.Spec.SentinelIsolation { + // Probe the informer cache before making any live rollout reads. The + // controller's Owns(NetworkPolicy) watch is required here: delayed + // owned grants enqueue another pass after becoming visible in cache. + ownedGrant := false + for _, suffix := range []string{"ingress", "egress"} { + var policy networkingv1.NetworkPolicy + if err := r.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-" + suffix}, &policy); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if metav1.IsControlledBy(&policy, gs) { + ownedGrant = true + } + } + if !ownedGrant { + return nil + } + // Workload reconciliation removes the opt-in labels by rolling both + // pod templates. Keep both grants until every live pod has switched + // back to the legacy policies, including during a partial rollout. + reader := r.APIReader + if reader == nil { + reader = r.Client + } + // An empty Pod list alone is insufficient: an old workload template + // could still create labeled replacements after that observation. + var game appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name}, &game); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if game.Spec.Template.Labels[sentinelIsolationLabel] == "true" || game.Status.ObservedGeneration < game.Generation || game.Status.CurrentRevision != game.Status.UpdateRevision { + return nil + } + var sentinel appsv1.Deployment + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &sentinel); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if sentinel.Spec.Template.Labels[sentinelIsolationLabel] == "true" || sentinel.Status.ObservedGeneration < sentinel.Generation || sentinel.Status.UpdatedReplicas != sentinel.Status.Replicas { + return nil + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(gs.Namespace), client.MatchingLabels{ + "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true", + }); err != nil { + return err + } + for i := range pods.Items { + name := pods.Items[i].Labels["app.kubernetes.io/name"] + if name == "gameplane-game" || name == "gameplane-waker" { + return nil + } + } + } + ports := networkPolicyPortsFromTemplate(tmpl) + for _, ingress := range []bool{true, false} { + suffix, target, peer := "egress", "gameplane-waker", "gameplane-game" + if ingress { + suffix, target, peer = "ingress", "gameplane-game", "gameplane-waker" + } + np := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-sentinel-" + suffix, Namespace: gs.Namespace}} + if !tmpl.Spec.SentinelIsolation || len(ports) == 0 { + if err := r.Get(ctx, client.ObjectKeyFromObject(np), np); err != nil { + if client.IgnoreNotFound(err) != nil { + return err + } + continue + } + if metav1.IsControlledBy(np, gs) { + uid := np.UID + if err := client.IgnoreNotFound(r.Delete(ctx, np, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}})); err != nil { + return err + } + } + continue + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { + if np.ResourceVersion != "" && !metav1.IsControlledBy(np, gs) { + return fmt.Errorf("sentinel policy %s is not owned by this GameServer", np.Name) + } + np.Spec = networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": target, "app.kubernetes.io/instance": gs.Name, + }}} + peers := []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": peer, "app.kubernetes.io/instance": gs.Name, + }}}} + if ingress { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeIngress} + np.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{From: peers, Ports: ports}} + } else { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} + np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{{To: peers, Ports: ports}} + } + return controllerutil.SetControllerReference(gs, np, r.Scheme) + }) + if err != nil { + return err + } + } + return nil +} + +// Invalid isolated templates must revoke old exposure before reporting failure. +// Read live so informer lag cannot preserve stale grants, and delete only the +// exact owned objects observed. Agent connectivity and workload state are kept. +func (r *GameServerReconciler) revokeInvalidSentinelIsolationExposure(ctx context.Context, gs *gameplanev1alpha1.GameServer) error { + objects := []client.Object{ + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gameIngressPolicyName(gs)}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-ingress"}}, + &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-sentinel-egress"}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: gs.Namespace, Name: gs.Name + "-game-direct"}}, + } + var failures []error + for _, object := range objects { + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + if !apierrors.IsNotFound(err) { + failures = append(failures, fmt.Errorf("read invalid-template exposure %s: %w", object.GetName(), err)) + } + continue + } + if !metav1.IsControlledBy(object, gs) { + continue + } + uid, version := object.GetUID(), object.GetResourceVersion() + if err := r.Delete(ctx, object, client.Preconditions{UID: &uid, ResourceVersion: &version}); client.IgnoreNotFound(err) != nil { + failures = append(failures, fmt.Errorf("revoke invalid-template exposure %s: %w", object.GetName(), err)) + } + } + return errors.Join(failures...) +} diff --git a/operator/internal/controller/sentinel_isolation_review_test.go b/operator/internal/controller/sentinel_isolation_review_test.go new file mode 100644 index 000000000..d3859e210 --- /dev/null +++ b/operator/internal/controller/sentinel_isolation_review_test.go @@ -0,0 +1,196 @@ +package controller + +import ( + "context" + "errors" + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +type isolationDrainReader struct { + client.Reader + workloadGets, podLists int +} + +func (r *isolationDrainReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + switch obj.(type) { + case *appsv1.StatefulSet, *appsv1.Deployment: + r.workloadGets++ + } + return r.Reader.Get(ctx, key, obj, opts...) +} +func (r *isolationDrainReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.PodList); ok { + r.podLists++ + } + return r.Reader.List(ctx, list, opts...) +} + +func TestSentinelOptOutWithoutOwnedGrantsSkipsLiveDrainReads(t *testing.T) { + for _, foreign := range []bool{false, true} { + s := testScheme(t) + _ = networkingv1.AddToScheme(s) + c := fake.NewClientBuilder().WithScheme(s).Build() + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + if foreign { + if err := c.Create(t.Context(), &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "alpha-sentinel-ingress", Namespace: gs.Namespace}}); err != nil { + t.Fatal(err) + } + } + reader := &isolationDrainReader{Reader: c} + r := &GameServerReconciler{Client: c, APIReader: reader, Scheme: s} + if err := r.reconcileSentinelIsolation(t.Context(), gs, singlePortTemplate(false, "none")); err != nil { + t.Fatal(err) + } + if reader.workloadGets != 0 || reader.podLists != 0 { + t.Fatal("settled opt-out performed live workload drainage reads") + } + } +} + +func TestSentinelOptOutCleansGrantsAfterDelayedCacheVisibility(t *testing.T) { + s := testScheme(t) + _ = networkingv1.AddToScheme(s) + c := fake.NewClientBuilder().WithScheme(s).Build() + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + hidden := true + r.Client = interceptor.NewClient(c, interceptor.Funcs{Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*networkingv1.NetworkPolicy); ok && hidden { + return apierrors.NewNotFound(schema.GroupResource{Resource: "networkpolicies"}, key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }}) + reader := &isolationDrainReader{Reader: c} + r.APIReader = reader + tmpl.Spec.SentinelIsolation = false + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + if reader.workloadGets != 0 || reader.podLists != 0 { + t.Fatal("cached absence did not take fast path") + } + // The owned NetworkPolicy watch enqueues after the informer makes its + // delayed grant visible. The next pass must perform the live safety proof. + hidden = false + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + if reader.workloadGets != 2 || reader.podLists != 1 { + t.Fatalf("visible grants bypassed live drainage proof: gets=%d lists=%d", reader.workloadGets, reader.podLists) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("delayed grants were never cleaned: %v %+v", err, policies.Items) + } +} + +func TestInvalidIsolatedTemplateRevokesOnlyOwnedExposure(t *testing.T) { + for _, mode := range []string{"owned", "foreign", "retryable-delete"} { + t.Run(mode, func(t *testing.T) { + s := testScheme(t) + _ = networkingv1.AddToScheme(s) + _ = batchv1.AddToScheme(s) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(false, "none") + tmpl.Name, tmpl.Spec.SentinelIsolation = "isolated", true + gs.Spec.TemplateRef.Name = tmpl.Name + owner := *metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer")) + if mode == "foreign" { + owner.UID = "foreign-server" + } + var exposures []client.Object + for _, name := range []string{gameIngressPolicyName(gs), gs.Name + "-sentinel-ingress", gs.Name + "-sentinel-egress"} { + exposures = append(exposures, &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name), OwnerReferences: []metav1.OwnerReference{owner}}}) + } + for _, name := range []string{gs.Name, gs.Name + "-game-direct"} { + exposures = append(exposures, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID(name), OwnerReferences: []metav1.OwnerReference{owner}}}) + } + one := int32(1) + workload := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace}, Spec: appsv1.StatefulSetSpec{Replicas: &one}} + agent := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-agent", Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}} + objects := append([]client.Object{gs, tmpl, workload, agent}, exposures...) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...).WithStatusSubresource(gs).Build() + deleteErr := errors.New("retryable deletion error") + cached := interceptor.NewClient(c, interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + switch obj.(type) { + case *networkingv1.NetworkPolicy, *corev1.Service: + return apierrors.NewNotFound(schema.GroupResource{Resource: "hidden-exposure"}, key.Name) + } + return cl.Get(ctx, key, obj, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if mode == "retryable-delete" && obj.GetName() == gameIngressPolicyName(gs) { + return deleteErr + } + options := (&client.DeleteOptions{}).ApplyOptions(opts) + if options.Preconditions == nil || options.Preconditions.UID == nil || options.Preconditions.ResourceVersion == nil { + t.Fatal("exposure deletion lacks UID/resourceVersion guard") + } + return cl.Delete(ctx, obj, opts...) + }, + }) + r := &GameServerReconciler{Client: cached, APIReader: c, Scheme: s} + revision, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: revision} + gs.Spec.TemplateRollout = &gameplanev1alpha1.TemplateRolloutSpec{Policy: "OnRequest"} + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + // A repaired catalog must not bypass validation of the still-invalid + // admitted snapshot or preserve its stale exposure. + tmpl.Spec.Ports = []gameplanev1alpha1.GamePort{{Name: "game", ContainerPort: 25565, Advertise: true}} + if err := c.Update(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + _, err = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}) + if mode == "retryable-delete" && !errors.Is(err, deleteErr) || mode != "retryable-delete" && err != nil { + t.Fatalf("cleanup error=%v", err) + } + for _, obj := range exposures { + err := c.Get(t.Context(), client.ObjectKeyFromObject(obj), obj) + keep := mode == "foreign" || (mode == "retryable-delete" && obj.GetName() == gameIngressPolicyName(gs)) + if keep && err != nil || !keep && !apierrors.IsNotFound(err) { + t.Fatalf("exposure %s retained=%t error=%v", obj.GetName(), keep, err) + } + } + var current appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(workload), ¤t); err != nil || !reflect.DeepEqual(current.Spec, workload.Spec) { + t.Fatalf("invalid template mutated workload: %v", err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(agent), agent); err != nil { + t.Fatal("invalid template removed agent Service", err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed { + t.Fatal("cleanup error concealed invalid-template failure") + } + }) + } +} diff --git a/operator/internal/controller/sentinel_isolation_test.go b/operator/internal/controller/sentinel_isolation_test.go new file mode 100644 index 000000000..570bcd07f --- /dev/null +++ b/operator/internal/controller/sentinel_isolation_test.go @@ -0,0 +1,244 @@ +package controller + +import ( + "context" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestSentinelIsolationPolicies(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := gameplanev1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = append(tmpl.Spec.Ports, + gameplanev1alpha1.GamePort{Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + gameplanev1alpha1.GamePort{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}) + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, target, peer string + ingress bool + }{ + {"alpha-sentinel-ingress", "gameplane-game", "gameplane-waker", true}, + {"alpha-sentinel-egress", "gameplane-waker", "gameplane-game", false}, + } { + var policy networkingv1.NetworkPolicy + if err := c.Get(t.Context(), client.ObjectKey{Namespace: "games", Name: tc.name}, &policy); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(&policy, gs) { + t.Fatal("missing server ownership") + } + selector := policy.Spec.PodSelector.MatchLabels + if selector["app.kubernetes.io/name"] != tc.target || selector["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("wrong target: %v", selector) + } + var peers []networkingv1.NetworkPolicyPeer + var ports []networkingv1.NetworkPolicyPort + if tc.ingress { + if len(policy.Spec.Ingress) != 1 || len(policy.Spec.Egress) != 0 { + t.Fatal("wrong ingress policy shape") + } + peers, ports = policy.Spec.Ingress[0].From, policy.Spec.Ingress[0].Ports + } else { + if len(policy.Spec.Egress) != 1 || len(policy.Spec.Ingress) != 0 { + t.Fatal("wrong egress policy shape") + } + peers, ports = policy.Spec.Egress[0].To, policy.Spec.Egress[0].Ports + } + if len(peers) != 1 || peers[0].NamespaceSelector != nil || peers[0].IPBlock != nil || peers[0].PodSelector.MatchLabels["app.kubernetes.io/name"] != tc.peer || peers[0].PodSelector.MatchLabels["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("cross-server peer grant: %+v", peers) + } + if len(ports) != 2 || ports[0].Port.IntVal != 25565 || *ports[0].Protocol != corev1.ProtocolTCP || ports[1].Port.IntVal != 19132 || *ports[1].Protocol != corev1.ProtocolUDP { + t.Fatalf("wrong protocol/port tuples: %+v", ports) + } + } + // Zero advertised ports removes both grants instead of opening all ports. + tmpl.Spec.Ports = nil + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("zero-port grants remain: %v %+v", err, policies.Items) + } + // Absent/default false preserves legacy behavior and creates no policies. + tmpl = singlePortTemplate(true, "minecraft") + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("legacy template changed: %v %+v", err, policies.Items) + } +} + +func TestSentinelIsolationDoesNotDeleteForeignPolicy(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = networkingv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "alpha-sentinel-ingress", Namespace: "games"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + if err := r.reconcileSentinelIsolation(t.Context(), gs, singlePortTemplate(false, "none")); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(foreign), foreign); err != nil { + t.Fatal("deleted foreign policy", err) + } +} + +type staleSentinelPodClient struct{ client.Client } + +func (c staleSentinelPodClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if pods, ok := list.(*corev1.PodList); ok { + pods.Items = nil + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func TestSentinelIsolationOptOutWaitsForBothLivePodLabels(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: staleSentinelPodClient{c}, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + pods := []*corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + } + for _, pod := range pods { + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + tmpl.Spec.SentinelIsolation = false + for remaining := 2; remaining >= 0; remaining-- { + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil { + t.Fatal(err) + } + want := 2 + if remaining == 0 { + want = 0 + } + if len(policies.Items) != want { + t.Fatalf("%d labeled pods remain: want %d grants, got %d", remaining, want, len(policies.Items)) + } + if remaining > 0 { + pod := pods[2-remaining] + delete(pod.Labels, sentinelIsolationLabel) + if err := c.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + } +} + +func TestSentinelIsolationRejectsNoAdvertisedPortsBeforeCreatingWorkloads(t *testing.T) { + scheme := testScheme(t) + _ = batchv1.AddToScheme(scheme) + _ = networkingv1.AddToScheme(scheme) + tmpl := singlePortTemplate(false, "none") + tmpl.Name = "isolated" + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = []gameplanev1alpha1.GamePort{{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + gs.Spec.TemplateRef.Name = tmpl.Name + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(gs).WithObjects(gs, tmpl).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + // Admission is a separate durable transition before template validation. + revision, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: revision} + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + condition := meta.FindStatusCondition(gs.Status.Conditions, "Ready") + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed || condition == nil || condition.Status != metav1.ConditionFalse || !strings.Contains(condition.Message, "sentinelIsolation requires at least one advertised port") { + t.Fatalf("missing actionable isolation failure: %+v", gs.Status) + } + var workloads appsv1.StatefulSetList + if err := c.List(t.Context(), &workloads); err != nil || len(workloads.Items) != 0 { + t.Fatalf("invalid template created workloads: %v %+v", err, workloads) + } +} + +func TestSentinelIsolationOptOutWaitsForWorkloadControllers(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + for _, test := range []struct { + name string + object client.Object + }{ + {"game-old-template", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"game-unobserved", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", Generation: 2}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 1}}}, + {"game-old-revision", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Status: appsv1.StatefulSetStatus{CurrentRevision: "old", UpdateRevision: "new"}}}, + {"sentinel-old-template", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"sentinel-unobserved", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games", Generation: 2}, Status: appsv1.DeploymentStatus{ObservedGeneration: 1}}}, + {"sentinel-old-replicas", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Status: appsv1.DeploymentStatus{Replicas: 2, UpdatedReplicas: 1}}}, + } { + t.Run(test.name, func(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.object).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + tmpl.Spec.SentinelIsolation = false + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 2 { + t.Fatalf("old workload could recreate labeled pods without grants: %v %+v", err, policies.Items) + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 06ca0b645..ffd7b191c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -288,6 +288,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. + - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. An opted-in template with zero advertised ports revokes its owned game-ingress and sentinel policies plus public/game-direct Services through uncached, UID/resourceVersion-guarded cleanup before reporting actionable `Ready=False`; it creates no workloads or unrestricted grants, preserves foreign resources and the agent Service, and reports the failure even when cleanup needs a retry. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. On opt-out, roll the labels off both workloads and retain the per-server grants until uncached reads confirm both controllers observed the unlabeled templates and completed their rollouts, and neither game nor sentinel pods still carry the label. Settled opt-outs first probe cached sentinel policies and skip live rollout reads when no owned grants exist. The owned NetworkPolicy watch must remain enabled: delayed cache visibility enqueues another cleanup pass. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. - Fixed-name child cleanup: Delete fixed-name objects (`-config` Secret, `-files` Secret, `-rcon` Secret, `-auto` BackupSchedule, `-wipe` Job) only when the GameServer controls them via OwnerReference; unowned objects with these names stay in place (same rule as game-ingress NetworkPolicy). Wipe cleanup carries the verified Job UID as a delete precondition; a foreign Job cannot acknowledge a pending wipe or resume its server. - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. Idle-transition and restart-acknowledgment conflicts, including wrapped errors, also return before logging and retry after one second with fresh state. Other errors retain their identity, error logging and controller-runtime backoff. The Module, NetworkCapture and Backup reconcilers apply the conflict rule to every error they return, via `requeueOnConflict` (`conflict.go`). - Load-balancer address management: translate spec.networking.addressPool / .address preferences onto the Service based on the cluster's address-manager flavor (MetalLB, Cilium, or none), report assignment status via the AddressAssignment condition. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index eca94e11c..ab46cee3b 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -436,8 +436,12 @@ EOF unbucketed() { :; } +bucket_network_isolation() { + printf '%s\n' TestNetworkIsolation_EnforcedAndUpgrade TestNetworkIsolation_ProbeOutput +} + bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads management-placement backup-preparation + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads network-isolation management-placement backup-preparation } list_bucket() { @@ -457,6 +461,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + network-isolation) bucket_network_isolation ;; restricted-workloads) bucket_restricted_workloads ;; management-placement) bucket_management_placement ;; backup-preparation) bucket_backup_preparation ;; diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index de2b09a51..5c4487b97 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation", + "telemetry", "optional-components", "restricted-workloads", "network-isolation", "management-placement", "backup-preparation", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go new file mode 100644 index 000000000..412b9b6ab --- /dev/null +++ b/test/e2e/network_isolation_e2e_test.go @@ -0,0 +1,230 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket uses Cilium, not kindnet. Exercise the effective union of all +// installed policies, including an existing legacy server's opt-in rollout. +func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { + ctx := t.Context() + const ns = "gameplane-games" + if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { + t.Fatalf("network-isolation bucket requires the enforcing Cilium fixture: %v", err) + } + stamp := time.Now().UnixNano() + legacyTemplate := fmt.Sprintf("net-legacy-%d", stamp) + isolatedTemplate := fmt.Sprintf("net-isolated-%d", stamp) + isolationTemplate(t, legacyTemplate, false) + isolationTemplate(t, isolatedTemplate, true) + a, b, c := fmt.Sprintf("net-a-%d", stamp), fmt.Sprintf("net-b-%d", stamp), fmt.Sprintf("net-c-%d", stamp) + for _, pair := range [][2]string{{a, legacyTemplate}, {b, legacyTemplate}, {c, isolatedTemplate}} { + applyBusyboxGameServer(t, ns, pair[0], pair[1]) + } + aIP := isolationGameIP(t, a, false) + bIP := isolationGameIP(t, b, false) + cIP := isolationGameIP(t, c, true) + probeA := isolationProbe(t, a, false) + probeC := isolationProbe(t, c, true) + + // Compatibility: absent-field templates keep their existing broad grant. + isolationHTTP(t, probeA, bIP, 25575, true) + isolationHTTP(t, probeC, cIP, 25566, true) + isolationHTTP(t, probeC, cIP, 25575, false) + isolationHTTP(t, probeC, aIP, 25566, false) + isolationHTTP(t, probeA, cIP, 25566, false) + + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldSelector := ss.Spec.Selector.DeepCopy() + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, legacyTemplate, types.MergePatchType, + []byte(`{"spec":{"sentinelIsolation":true}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + aIP, bIP = isolationGameIP(t, a, true), isolationGameIP(t, b, true) + ss, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(oldSelector, ss.Spec.Selector) { + t.Fatalf("opt-in must preserve the immutable StatefulSet selector: %v", err) + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, probeA, types.MergePatchType, + []byte(`{"metadata":{"labels":{"gameplane.local/sentinel-isolation":"true"}}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + isolationHTTP(t, probeA, aIP, 25566, true) + isolationHTTP(t, probeA, aIP, 25575, false) + isolationHTTP(t, probeA, bIP, 25566, false) + isolationHTTP(t, probeA, bIP, 25575, false) + + // Kubelet HTTP readiness and agent heartbeats still work with strict CIDRs. + // Also exercise an actual API -> agent mTLS call through its Service. + envInstance.BootstrapAdmin(t, adminUsername, adminPassword) + cli := envInstance.APIClient(t, adminUsername, adminPassword) + defer cli.Close() + requireAgentReady(t, ns, a) + waitAgentReachable(t, cli, a) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + hb, _, _ := unstructured.NestedString(gs.Object, "status", "agent", "lastHeartbeat") + stamp, err := time.Parse(time.RFC3339, hb) + return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" + }) +} + +func isolationTemplate(t *testing.T, name string, isolated bool) { + t.Helper() + spec := map[string]any{ + "displayName": name, "game": "busybox", "version": "1", "image": "busybox:1.36", + "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, + "ports": []any{ + map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}, + map[string]any{"name": "management", "containerPort": int64(25575), "protocol": "TCP", "advertise": false}, + }, + "probes": map[string]any{"readiness": map[string]any{ + "httpGet": map[string]any{"path": "/", "port": int64(25566)}, "periodSeconds": int64(2), + }}, + } + if isolated { + spec["sentinelIsolation"] = true + } + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameTemplate", + "metadata": map[string]any{"name": name}, "spec": spec, + }} + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Create(t.Context(), obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) +} + +func isolationGameIP(t *testing.T, name string, isolated bool) string { + t.Helper() + var ip string + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if pod.DeletionTimestamp != nil || (pod.Labels["gameplane.local/sentinel-isolation"] == "true") != isolated { + return false, "waiting for rollout with expected isolation label" + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { + ip = pod.Status.PodIP + return true, "" + } + } + return false, "waiting for HTTP readiness probe" + }) + return ip +} + +// A synthetic client carries the exact real sentinel selectors. It exposes no +// listener and owns no Service/Deployment, so the running game is unaffected. +func isolationProbe(t *testing.T, server string, isolated bool) string { + t.Helper() + const ns = "gameplane-games" + name := server + "-probe" + labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} + if isolated { + labels["gameplane.local/sentinel-isolation"] = "true" + } + noToken := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, + Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, + Containers: []corev1.Container{{Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}}}}, + } + if _, err := envInstance.K8s.CoreV1().Pods(ns).Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Pods(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + got, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return got.Status.Phase == corev1.PodRunning, "waiting for probe pod" + }) + return name +} + +func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { + t.Helper() + // Every exec must succeed regardless of network outcome. Otherwise a broken + // kubectl/CNI fixture could masquerade as the expected denied connection. + target := fmt.Sprintf("http://%s:%d/", ip, port) + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + out, err := envInstance.Kubectl(ctx, "exec", "-n", "gameplane-games", source, "--", + "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) + if err != nil { + return false, "probe execution failed: " + err.Error() + out + } + got := isolationProbeVerdict(out) + want := "DENIED" + if allowed { + want = "ALLOWED" + } + return got == want, fmt.Sprintf("%s -> %s: got %q, want %s; output: %q", source, target, got, want, out) + }) +} + +// Kubectl combines stdout and stderr, so wget's expected timeout diagnostic +// can precede the result. Require one exact marker; never accept missing or +// contradictory results, and retain the full output in the failure message. +func isolationProbeVerdict(out string) string { + var verdict string + for line := range strings.SplitSeq(out, "\n") { + line = strings.TrimSpace(line) + if line != "ALLOWED" && line != "DENIED" { + continue + } + if verdict != "" { + return "" + } + verdict = line + } + return verdict +} + +func TestNetworkIsolation_ProbeOutput(t *testing.T) { + t.Parallel() + for _, tc := range []struct{ name, output, want string }{ + {"allowed", "ALLOWED\n", "ALLOWED"}, + {"denied with wget timeout", "wget: download timed out\nDENIED\n", "DENIED"}, + {"denied with carriage return", "wget: connection refused\r\nDENIED\r\n", "DENIED"}, + {"diagnostic only", "wget: download timed out\n", ""}, + {"marker substring", "unexpected ALLOWED response\n", ""}, + {"contradictory markers", "ALLOWED\nDENIED\n", ""}, + {"duplicate markers", "DENIED\nDENIED\n", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := isolationProbeVerdict(tc.output); got != tc.want { + t.Fatalf("probe verdict = %q, want %q for %q", got, tc.want, tc.output) + } + }) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 0fdf94e2c..82e57628d 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -6,6 +6,13 @@ ## Purpose +The amd64 `network-isolation` bucket requires the Cilium fixture +(`GAMEPLANE_E2E_CNI=cilium`) and strict chart CIDRs derived from the single-node +cluster. It tests actual legacy versus isolated sentinel connections, an +existing template's opt-in rollout without selector changes, HTTP kubelet +readiness, fresh agent heartbeats, and API-to-agent mTLS access. It uses one +admin login and runs separately from the kindnet shape-only checks. + ### Restricted workload drill (#714) The dedicated `restricted-workloads` bucket runs on amd64 and arm64. It @@ -111,10 +118,10 @@ The golangci-lint job mirrors this: it must be invoked with `--build-tags=e2e` o ## The bucket contract — the module's most important invariant -The `management-placement` audit bucket is registered in the change-based selector, -with explicit component mapping and an amd64-only selection-summary row. Its +The `network-isolation` and `management-placement` audit buckets are registered in the change-based selector, +with explicit component mapping and amd64-only selection-summary rows. Each matrix leg skips cluster setup only when its selected test pattern is empty; -full-scope and affected-component runs must include it. +full-scope and affected-component runs must include the affected buckets. `buckets.sh` is the single source of truth for how CI's e2e suite is sharded into parallel jobs (`operator`, `api-auth`, `api-roles`, `api-rbac`, `api-agent`, `api-mods`, `ratelimit`, `bot-fast`, `bot-heavy`, `multicluster`, `upgrade`). Each bucket is a flat list of **exact Go function names** (e.g. `TestGameServer_MinecraftJavaBot_Joined`), and `buckets.sh regex ` turns that list into an **anchored** `^(A|B|...)$` regex passed to `go test -run`. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index f5297e1ed..c247bcf4b 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent"] }, { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestHelmInstall_", "components": ["*"] },