From 1d63654bfec14175c4bd1a4443983afcde2b9690 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Tue, 10 Nov 2026 20:52:43 -0500
Subject: [PATCH 1/6] feat(observability): discover workloads in managed
 namespaces and alert on missing targets

Signed-off-by: Baylem <zelathis@gmail.com>
---
 .../gameplane/templates/monitoring-rules.yaml | 40 +++++++++++++------
 .../gameplane/templates/servicemonitors.yaml  |  4 ++
 charts/gameplane/tests/monitoring-rules.sh    |  8 +++-
 .../tests/monitoring-rules.test.yaml          | 27 +++++++++++++
 docs/monitoring.md                            | 14 +++++++
 5 files changed, 79 insertions(+), 14 deletions(-)

diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml
index 98b383683..dbf543860 100644
--- a/charts/gameplane/templates/monitoring-rules.yaml
+++ b/charts/gameplane/templates/monitoring-rules.yaml
@@ -1,4 +1,5 @@
 {{- if and .Values.prometheusRules.enabled (eq (include "gameplane.operatorEnabled" .) "true") }}
+{{- $namespaces := join "|" (include "gameplane.operatorGameNamespaces" . | fromJsonArray) }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -13,59 +14,72 @@ spec:
   groups:
     - name: gameplane.workloads
       rules:
+        - alert: GameplaneScrapeTargetMissing
+          expr: |
+            (
+              max by (namespace, pod, gameplane_component) (
+                label_replace(kube_pod_container_status_running{namespace=~"{{ $namespaces }}",container=~"agent|tunnel"} == 1,
+                  "gameplane_component", "$1", "container", "(agent|tunnel)")
+              )
+              and on (namespace, pod) kube_pod_labels{namespace=~"{{ $namespaces }}",label_app_kubernetes_io_name=~"gameplane-game|gameplane-tunnel"}
+            )
+            unless on (namespace, pod, gameplane_component) up{namespace=~"{{ $namespaces }}",gameplane_component=~"agent|tunnel"}
+          for: 5m
+          labels: { severity: warning }
+          annotations: { summary: "Running Gameplane workload is missing from scrape discovery" }
         - alert: GameplaneScrapeUnavailable
-          expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace="{{ .Values.gamesNamespace }}"}) == 0
+          expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace=~"{{ $namespaces }}"}) == 0
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Gameplane metrics target cannot be scraped" }
         - alert: GameplaneRelayUnavailable
-          expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 1)
+          expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 1)
           for: 2m
           labels: { severity: warning }
           annotations: { summary: "Gameplane relay connection or forwards are unavailable" }
         - alert: GameplaneRelayHealthUnknown
-          expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace="{{ .Values.gamesNamespace }}"}) == 1)
+          expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace=~"{{ $namespaces }}"}) == 1)
           for: 5m
           labels: { severity: info }
           annotations: { summary: "Gameplane provider cannot confirm live relay health" }
         - alert: GameplanePerformanceUnknown
-          expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace="{{ .Values.gamesNamespace }}"}) == 0
+          expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace=~"{{ $namespaces }}"}) == 0
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Declared game performance measurement is unavailable" }
         - alert: GameplanePerformanceDegraded
           expr: |
-            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"})
+            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"})
               < on (namespace, pod, metric, signal, unit)
-             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="below"}))
+             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="below"}))
             or
-            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"})
+            (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"})
               > on (namespace, pod, metric, signal, unit)
-             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="above"}))
+             max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="above"}))
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "Game performance crossed its module-declared threshold" }
         - alert: GameplaneCPUThrottled
-          expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m])) > 0.25
+          expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m])) > 0.25
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Game container CPU is throttled in more than 25% of periods" }
         - alert: GameplaneMemoryPressure
-          expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} / (container_spec_memory_limit_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} > 0)) > 0.9
+          expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"{{ $namespaces }}",container="game"} / (container_spec_memory_limit_bytes{namespace=~"{{ $namespaces }}",container="game"} > 0)) > 0.9
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Game container working set exceeds 90% of its memory limit" }
         - alert: GameplaneContainerOOM
-          expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="{{ .Values.gamesNamespace }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace="{{ .Values.gamesNamespace }}",reason="OOMKilled"}) == 1)
+          expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace=~"{{ $namespaces }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $namespaces }}",reason="OOMKilled"}) == 1)
           labels: { severity: warning }
           annotations: { summary: "Gameplane container restarted after an OOM kill" }
         - alert: GameplanePodPending
-          expr: max by (namespace, pod) (kube_pod_status_phase{namespace="{{ .Values.gamesNamespace }}",phase="Pending"}) == 1
+          expr: max by (namespace, pod) (kube_pod_status_phase{namespace=~"{{ $namespaces }}",phase="Pending"}) == 1
           for: 10m
           labels: { severity: warning }
           annotations: { summary: "Gameplane Pod has remained Pending for ten minutes" }
         - alert: GameplaneNodePressure
-          expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace="{{ .Values.gamesNamespace }}"})
+          expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace=~"{{ $namespaces }}"})
           for: 5m
           labels: { severity: warning }
           annotations: { summary: "A node hosting Gameplane Pods reports resource pressure" }
diff --git a/charts/gameplane/templates/servicemonitors.yaml b/charts/gameplane/templates/servicemonitors.yaml
index 2b5b6834b..2ae0b26e4 100644
--- a/charts/gameplane/templates/servicemonitors.yaml
+++ b/charts/gameplane/templates/servicemonitors.yaml
@@ -101,6 +101,8 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
+  namespaceSelector:
+    matchNames: {{ include "gameplane.operatorGameNamespaces" . }}
   selector:
     matchLabels: { app.kubernetes.io/name: gameplane-game }
   podMetricsEndpoints:
@@ -124,6 +126,8 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
 spec:
+  namespaceSelector:
+    matchNames: {{ include "gameplane.operatorGameNamespaces" . }}
   selector:
     matchLabels: { app.kubernetes.io/name: gameplane-tunnel }
   podMetricsEndpoints:
diff --git a/charts/gameplane/tests/monitoring-rules.sh b/charts/gameplane/tests/monitoring-rules.sh
index 6e39efd8e..ae876184e 100644
--- a/charts/gameplane/tests/monitoring-rules.sh
+++ b/charts/gameplane/tests/monitoring-rules.sh
@@ -4,14 +4,20 @@ set -euo pipefail
 cd "$(dirname "$0")/../../.."
 scratch="$(mktemp -d)"
 trap 'rm -rf "$scratch"' EXIT
-helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml > "$scratch/rendered.yaml"
+helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml \
+  --set 'operator.gameNamespaces={tenant-b,gameplane-games}' > "$scratch/rendered.yaml"
 python3 - "$scratch" <<'PY'
 import pathlib,sys,yaml
 root=pathlib.Path(sys.argv[1])
 groups=[]
+monitors={}
 for document in yaml.safe_load_all((root/'rendered.yaml').read_text()):
     if document and document.get('kind')=='PrometheusRule':
         groups.extend(document['spec']['groups'])
+    if document and document.get('kind')=='PodMonitor':
+        monitors[document['metadata']['name']]=document['spec']['namespaceSelector']['matchNames']
+expected=['gameplane-games','tenant-b']
+assert monitors=={'gameplane-agent':expected,'gameplane-tunnel':expected}, monitors
 (root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups}))
 PY
 cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml"
diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml
index 2d285dedf..51c6153a3 100644
--- a/charts/gameplane/tests/monitoring-rules.test.yaml
+++ b/charts/gameplane/tests/monitoring-rules.test.yaml
@@ -1,6 +1,33 @@
 rule_files: [rules.yaml]
 evaluation_interval: 1m
 tests:
+  - interval: 1m
+    input_series:
+      - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="sleepy-0",container="agent"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="tenant-b",pod="sleepy-0",label_app_kubernetes_io_name="gameplane-game"}'
+        values: '1x12'
+      - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="relay",container="tunnel"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="tenant-b",pod="relay",label_app_kubernetes_io_name="gameplane-tunnel"}'
+        values: '1x12'
+      - series: 'up{namespace="tenant-b",pod="relay",gameplane_component="tunnel"}'
+        values: '0x12'
+      - series: 'kube_pod_container_status_running{namespace="unmanaged",pod="hidden",container="agent"}'
+        values: '1x12'
+      - series: 'kube_pod_labels{namespace="unmanaged",pod="hidden",label_app_kubernetes_io_name="gameplane-game"}'
+        values: '1x12'
+    alert_rule_test:
+      - eval_time: 8m
+        alertname: GameplaneScrapeTargetMissing
+        exp_alerts:
+          - exp_labels: {namespace: tenant-b, pod: sleepy-0, gameplane_component: agent, severity: warning}
+            exp_annotations: {summary: "Running Gameplane workload is missing from scrape discovery"}
+      - eval_time: 8m
+        alertname: GameplaneScrapeUnavailable
+        exp_alerts:
+          - exp_labels: {namespace: tenant-b, pod: relay, gameplane_component: tunnel, severity: warning}
+            exp_annotations: {summary: "Gameplane metrics target cannot be scraped"}
   - interval: 1m
     input_series:
       - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds"}'
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 83a22b300..5f5659831 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -28,6 +28,20 @@ Check Prometheus **Status → Targets** for agent and tunnel targets with `up=1`
 then query `gameplane_agent_players_online` and `gameplane_tunnel_ready`.
 Unknown player counts are absent even when scraping works.
 
+The agent and tunnel PodMonitors live in the primary games namespace. Their
+target lists include that namespace and every entry in `operator.gameNamespaces`.
+Ensure Prometheus honors PodMonitor namespace selectors and can list and watch
+pods in each target namespace. Provision the same approved scraper ingress in
+additional namespaces that enforce default deny, and allow Prometheus egress.
+Workload alerts select the same explicit list of namespaces.
+
+`GameplaneScrapeUnavailable` fires when a discovered endpoint stays down.
+`GameplaneScrapeTargetMissing` compares discovered targets with running
+agent/tunnel containers on Gameplane-labeled pods. Enable kube-state-metrics
+pod label collection for `app.kubernetes.io/name` (for example
+`--metric-labels-allowlist=pods=[app.kubernetes.io/name]`) and scrape its
+container-running and pod-label metrics to enable that comparison.
+
 With the operator and its NetworkPolicies enabled, the chart requires a nonempty
 `serviceMonitors.scrapeNamespaceSelector`. Missing configuration fails with a
 message naming the setting. If another system owns the policies, set

From 0a0fc729ba484339236ae2f8203d708142e8e9e3 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Wed, 11 Nov 2026 00:27:49 -0500
Subject: [PATCH 2/6] feat(observability): verify tenant scrapes and expose
 relay saturation

Signed-off-by: Baylem <zelathis@gmail.com>
---
 .../dashboards/gameplane-workloads.json       | 40 ++++++++++
 .../gameplane/templates/monitoring-rules.yaml |  7 ++
 .../tests/monitoring-rules.test.yaml          | 21 +++++
 deploy/kind/monitoring.sh                     |  7 +-
 docs/monitoring.md                            | 26 ++++++
 test/e2e/monitoring_e2e_test.go               | 80 ++++++++++++++++++-
 test/e2e/specs.md                             |  2 +-
 7 files changed, 177 insertions(+), 6 deletions(-)

diff --git a/charts/gameplane/dashboards/gameplane-workloads.json b/charts/gameplane/dashboards/gameplane-workloads.json
index f0a978b26..cc0b43522 100644
--- a/charts/gameplane/dashboards/gameplane-workloads.json
+++ b/charts/gameplane/dashboards/gameplane-workloads.json
@@ -340,6 +340,46 @@
           "legendFormat": "{{namespace}} / {{pod}} {{metric}}"
         }
       ]
+    },
+    {
+      "id": 11,
+      "title": "Relay drops and rejected flows",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": { "defaults": { "unit": "ops", "noValue": "Unknown", "custom": { "spanNulls": false } } },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (namespace,pod,protocol) (rate(gameplane_tunnel_flow_rejections_total{namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{namespace}} / {{pod}} {{protocol}} rejections"
+        },
+        {
+          "refId": "B",
+          "expr": "max by (namespace,pod) (rate(gameplane_tunnel_udp_queue_drops_total{namespace=~\"$namespace\"}[5m]))",
+          "legendFormat": "{{namespace}} / {{pod}} UDP queue drops"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Game process RSS and container working set",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
+      "datasource": { "type": "prometheus", "uid": "${datasource}" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "noValue": "Unknown", "custom": { "spanNulls": false } } },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "max by (namespace,pod) (gameplane_agent_memory_bytes{namespace=~\"$namespace\"})",
+          "legendFormat": "{{namespace}} / {{pod}} process RSS"
+        },
+        {
+          "refId": "B",
+          "expr": "max by (namespace,pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"game\"})",
+          "legendFormat": "{{namespace}} / {{pod}} working set"
+        }
+      ]
     }
   ]
 }
diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml
index dbf543860..181b0d16f 100644
--- a/charts/gameplane/templates/monitoring-rules.yaml
+++ b/charts/gameplane/templates/monitoring-rules.yaml
@@ -42,6 +42,13 @@ spec:
           for: 5m
           labels: { severity: info }
           annotations: { summary: "Gameplane provider cannot confirm live relay health" }
+        - alert: GameplaneRelaySaturated
+          expr: |
+            max by (namespace, pod) (increase(gameplane_tunnel_flow_rejections_total{namespace=~"{{ $namespaces }}"}[5m])) > 0
+            or
+            max by (namespace, pod) (increase(gameplane_tunnel_udp_queue_drops_total{namespace=~"{{ $namespaces }}"}[5m])) > 0
+          labels: { severity: warning }
+          annotations: { summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets" }
         - alert: GameplanePerformanceUnknown
           expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace=~"{{ $namespaces }}"}) == 0
           for: 5m
diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml
index 51c6153a3..d272e8a6f 100644
--- a/charts/gameplane/tests/monitoring-rules.test.yaml
+++ b/charts/gameplane/tests/monitoring-rules.test.yaml
@@ -1,6 +1,27 @@
 rule_files: [rules.yaml]
 evaluation_interval: 1m
 tests:
+  - interval: 1m
+    input_series:
+      - series: 'gameplane_tunnel_flow_rejections_total{namespace="gameplane-games",pod="busy",protocol="tcp"}'
+        values: '0x2 1x10'
+      - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="tenant-b",pod="backlogged"}'
+        values: '0x2 1x10'
+      - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="unmanaged",pod="outside"}'
+        values: '0+10x12'
+      - series: 'gameplane_tunnel_forward_metrics_supported{namespace="tenant-b",pod="opaque"}'
+        values: '0x12'
+    alert_rule_test:
+      - eval_time: 4m
+        alertname: GameplaneRelaySaturated
+        exp_alerts:
+          - exp_labels: {namespace: gameplane-games, pod: busy, severity: warning}
+            exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"}
+          - exp_labels: {namespace: tenant-b, pod: backlogged, severity: warning}
+            exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"}
+      - eval_time: 10m
+        alertname: GameplaneRelaySaturated
+        exp_alerts: []
   - interval: 1m
     input_series:
       - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="sleepy-0",container="agent"}'
diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh
index f5c2e4a4e..4bc8e3dbf 100644
--- a/deploy/kind/monitoring.sh
+++ b/deploy/kind/monitoring.sh
@@ -18,8 +18,13 @@ sed 's/namespace: default/namespace: monitoring/g' "$scratch/bundle.yaml" > "$sc
 kubectl apply --server-side -f "$scratch/monitoring.yaml"
 kubectl wait --for=condition=Established --timeout=120s crd/prometheuses.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com
 kubectl rollout status -n monitoring deployment/prometheus-operator --timeout=180s
+kubectl create namespace gameplane-monitoring-b --dry-run=client -o yaml | kubectl apply -f -
 helm --kube-context "$context" upgrade gameplane charts/gameplane --namespace gameplane-system --reuse-values \
-  -f docs/examples/monitoring/gameplane-values.yaml --wait --timeout 5m
+  -f docs/examples/monitoring/gameplane-values.yaml \
+  --set 'operator.gameNamespaces[0]=gameplane-monitoring-b' \
+  --set networkPolicies.tenantIsolation=true --set networkPolicies.probePorts=null \
+  --set networkPolicies.gameEgress.enabled=false --set networkPolicies.backupEgress.enabled=false \
+  --wait --timeout 5m
 kubectl apply -f docs/examples/monitoring/prometheus.yaml
 kubectl wait -n monitoring --for=create statefulset/prometheus-gameplane --timeout=120s
 kubectl rollout status -n monitoring statefulset/prometheus-gameplane --timeout=180s
diff --git a/docs/monitoring.md b/docs/monitoring.md
index d35fe2a6f..a6d21f6c0 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -35,6 +35,12 @@ pods in each target namespace. Provision the same approved scraper ingress in
 additional namespaces that enforce default deny, and allow Prometheus egress.
 Workload alerts select the same explicit list of namespaces.
 
+For installations using `networkPolicies.tenantIsolation: true`, the chart
+installs the approved scraper ingress in every managed game namespace. Tenant
+operators should query a tenant-scoped Prometheus endpoint or a datasource
+that enforces their namespace. A Grafana variable filters a view and grants no
+authorization. Keep the fleet-wide Prometheus credentials with operators.
+
 `GameplaneScrapeUnavailable` fires when a discovered endpoint stays down.
 `GameplaneScrapeTargetMissing` compares discovered targets with running
 agent/tunnel containers on Gameplane-labeled pods. Enable kube-state-metrics
@@ -149,6 +155,12 @@ staleness limit. See [relay-health.md](relay-health.md).
 Tailscale, frp and Playit all use the same metrics path. Metrics contain no
 credentials, login URLs, proxy names or provider error messages.
 
+`GameplaneRelaySaturated` fires on an increase in a supervisor flow-rejection
+or UDP queue-drop counter over five minutes. The dashboard shows both rates
+by namespace and Pod. The embedded Tailscale forwarder supplies these counters.
+For frp and Playit the support gauge is zero and the counters are absent.
+Use provider-side measurements for those paths.
+
 Container CPU throttling and working-set pressure require kubelet/cAdvisor
 scrapes. OOM/restart, Pending Pod and node pressure rules require kube-state-metrics.
 Install/configure those collectors (kube-prometheus-stack normally provides
@@ -159,6 +171,20 @@ container memory. These rules filter the games namespace and use `max` to
 deduplicate replicated sources. The existing fleet rules likewise collapse
 duplicate operator replicas instead of adding their cache-derived counts.
 
+In operator-managed games the RSS series sums the game processes in the shared
+PID namespace. cAdvisor reports the game container's working set independently.
+Distinguish sustained growth from allocator churn by correlating both with an
+engine profiler, such as JVM allocation and GC events from Java Flight Recorder
+on a compatible Minecraft JVM. Allocation rate can rise while RSS stays flat.
+Native engine heap or frame profiles require that engine's profiling tools.
+
+At official module revision `48b8b65c`, the Minecraft Java, ARK Survival
+Ascended, Palworld, Project Zomboid and CS2 templates have no numeric tick
+or frame declarations. Minecraft Java's `seed` readout is display-only. Use
+a module version with an engine-verified command, response and unit before
+setting a performance alert for that game. A server tick-rate setting describes
+its configured target, while the time series must describe observed ticks.
+
 The Workloads dashboard shows scrape health, relay readiness/known state,
 supported game measurements, container pressure, restarts and Pending Pods.
 Missing measurements remain gaps. CI renders and evaluates the shipped alert
diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go
index 00bdb02ae..1456d1945 100644
--- a/test/e2e/monitoring_e2e_test.go
+++ b/test/e2e/monitoring_e2e_test.go
@@ -17,6 +17,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 )
 
@@ -72,17 +73,19 @@ import json,pathlib,threading
 value=pathlib.Path('/tmp/performance')
 count=pathlib.Path('/tmp/performance-queries')
 value.write_text('20')
+heap=[]
 count.write_text('0')
 class Handler(BaseHTTPRequestHandler):
     def log_message(self,*args): pass
     def do_GET(self):
+        if self.path=='/allocate': heap.append(bytearray(16*1024*1024))
         self.send_response(200); self.end_headers(); self.wfile.write(b'ok')
     def do_POST(self):
         command=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0')))).get('command','')
         if command=='stats':
             count.write_text(str(int(count.read_text())+1))
             output='TPS: '+value.read_text()+' Tick: 50'
-        else: output='0 players online'
+        else: output='unavailable' if pathlib.Path('/tmp/players-unknown').exists() else '0 players online'
         self.send_response(200); self.end_headers(); self.wfile.write(json.dumps({'output':output}).encode())
 threading.Thread(target=ThreadingHTTPServer(('127.0.0.1',25575),Handler).serve_forever,daemon=True).start()
 ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
@@ -103,6 +106,15 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 		_ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{})
 	})
 	gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}}
+	const otherTenant = "gameplane-monitoring-b"
+	other := gs.DeepCopy()
+	other.SetNamespace(otherTenant)
+	if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Create(ctx, other, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		_ = envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Delete(context.Background(), name, metav1.DeleteOptions{})
+	})
 	gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{})
 	if err != nil {
 		t.Fatal(err)
@@ -120,10 +132,35 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 	}
 	selector := fmt.Sprintf(`namespace="gameplane-games",server="%s"`, name)
 	waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1")
+	otherSelector := fmt.Sprintf(`namespace="%s",server="%s"`, otherTenant, name)
+	waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1")
+	// Invalidate the player response on one of two identically named games.
+	waitQuery(`max(gameplane_agent_players_online{`+otherSelector+`})`, "0")
+	if out, err := envInstance.Kubectl(ctx, "exec", "-n", otherTenant, name+"-0", "-c", "game", "--", "touch", "/tmp/players-unknown"); err != nil {
+		t.Fatalf("invalidate player count: %v %s", err, out)
+	}
+	waitQuery(`absent(gameplane_agent_players_online{`+otherSelector+`})`, "1")
+	waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1")
+	waitQuery(`max(gameplane_agent_players_online{`+selector+`})`, "0")
+	waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+otherSelector+`})`, "20")
 	waitQuery(`max(up{gameplane_component="tunnel",`+selector+`})`, "1")
 	waitQuery(`max(gameplane_tunnel_ready{`+selector+`})`, "1")
 	waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, "20")
 	waitQuery(`max(gameplane_agent_performance_value{metric="tick",`+selector+`})`, "0.05")
+	rssQuery := `max(gameplane_agent_memory_bytes{` + selector + `})`
+	waitQuery(rssQuery+` > bool 0`, "1")
+	baseline, err := query(rssQuery)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rss, err := strconv.ParseFloat(baseline, 64)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `import urllib.request; urllib.request.urlopen('http://127.0.0.1:25566/allocate').read()`); err != nil {
+		t.Fatalf("allocate memory in game process: %v %s", err, out)
+	}
+	waitQuery(fmt.Sprintf(`%s > bool %.0f`, rssQuery, rss+8*1024*1024), "1")
 	waitQuery(`count(gameplane_agent_performance_value{metric="display",`+selector+`}) or vector(0)`, "0")
 	readCount := func() int {
 		out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; print(Path('/tmp/performance-queries').read_text())`)
@@ -140,6 +177,39 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 	if err != nil {
 		t.Fatal(err)
 	}
+	otherProbe := name + "-other"
+	monitoringProbe(t, otherTenant, otherProbe)
+	// Remove one tenant from PodMonitor discovery while its pod remains ready.
+	monitors := envInstance.Dyn.Resource(schema.GroupVersionResource{Group: "monitoring.coreos.com", Version: "v1", Resource: "podmonitors"}).Namespace(ocGamesNS)
+	monitor, err := monitors.Get(ctx, "gameplane-agent", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	names, _, err := unstructured.NestedStringSlice(monitor.Object, "spec", "namespaceSelector", "matchNames")
+	if err != nil {
+		t.Fatal(err)
+	}
+	patchNamespaces := func(ctx context.Context, namespaces []string) error {
+		body, err := json.Marshal(map[string]any{"spec": map[string]any{"namespaceSelector": map[string]any{"matchNames": namespaces}}})
+		if err != nil {
+			return err
+		}
+		_, err = monitors.Patch(ctx, "gameplane-agent", types.MergePatchType, body, metav1.PatchOptions{})
+		return err
+	}
+	t.Cleanup(func() { _ = patchNamespaces(context.Background(), names) })
+	if err := patchNamespaces(ctx, []string{ocGamesNS}); err != nil {
+		t.Fatal(err)
+	}
+	waitQuery(`absent(up{gameplane_component="agent",`+otherSelector+`})`, "1")
+	if ready, err := envInstance.PodIsReady(ctx, otherTenant, name+"-0"); err != nil || !ready {
+		t.Fatalf("missing target pod ready=%t err=%v", ready, err)
+	}
+	waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1")
+	if err := patchNamespaces(ctx, names); err != nil {
+		t.Fatal(err)
+	}
+	waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1")
 	approvedProbe := name + "-approved"
 	monitoringProbe(t, "monitoring", approvedProbe)
 	before := readCount()
@@ -195,9 +265,11 @@ try:
 except (OSError,TimeoutError):
     print('DENIED')
 `
-	out, err := envInstance.Kubectl(ctx, "exec", "-n", ocNS, probeName, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics")
-	if err != nil || strings.TrimSpace(out) != "DENIED" {
-		t.Fatalf("unapproved namespace reached metrics, or probe failed: %v %s", err, out)
+	for namespace, probe := range map[string]string{ocNS: probeName, otherTenant: otherProbe} {
+		out, err := envInstance.Kubectl(ctx, "exec", "-n", namespace, probe, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics")
+		if err != nil || strings.TrimSpace(out) != "DENIED" {
+			t.Fatalf("unapproved namespace %s reached metrics, or probe failed: %v %s", namespace, err, out)
+		}
 	}
 }
 
diff --git a/test/e2e/specs.md b/test/e2e/specs.md
index 0f300c26e..b3d5d30ca 100644
--- a/test/e2e/specs.md
+++ b/test/e2e/specs.md
@@ -57,7 +57,7 @@ stop, status reset/leader recovery, exact request binding, snapshot integrity,
 complete pinned render inputs and source-template preservation for CSI restore.
 
 - Keep the lint workflow, devcontainer and `lint-gate-verify.sh` on the exact `v2.14.0` linter pin, whose importer supports Go 1.27.3 export data. The verifier's valid fixtures use the same pin; regression fixtures reject `latest` and a stale fixed version in any lint step.
-- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It discovers real agent/tunnel targets through the documented chart values, verifies cached numeric samples, valid zero, invalid-value omission, no scrape-driven commands, policy revocation/recovery and refusal from an unapproved namespace. It performs no API login and does not opt in external modules.
+- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It runs identically named games in two tenant namespaces, removes and restores one discovery target, revokes and restores scrape ingress, and verifies that a tenant and a management Pod cannot read another tenant's metrics. It checks zero versus unknown player counts, game-process memory growth, numeric tick samples and invalid-value omission. Repeated scrapes must use the interval cache. The game fixture implements deterministic RCON responses and requires no external module or API login.
 
 - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced.
 

From 4340be059097b5f99649eb2426fe36a1d91559e3 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Wed, 11 Nov 2026 01:07:32 -0500
Subject: [PATCH 3/6] docs(observability): link engine-verified Palworld frame
 rate support

Signed-off-by: Baylem <zelathis@gmail.com>
---
 docs/monitoring.md | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/docs/monitoring.md b/docs/monitoring.md
index a6d21f6c0..e11e2896c 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -167,7 +167,7 @@ Install/configure those collectors (kube-prometheus-stack normally provides
 them); the minimal Prometheus example demonstrates Gameplane discovery and
 does not deploy them or Grafana. A missing upstream collector leaves those
 series absent; it does not mean zero pressure. Agent process RSS is not total
-container memory. These rules filter the games namespace and use `max` to
+container memory. These rules filter the explicitly managed game namespaces and use `max` to
 deduplicate replicated sources. The existing fleet rules likewise collapse
 duplicate operator replicas instead of adding their cache-derived counts.
 
@@ -178,12 +178,17 @@ engine profiler, such as JVM allocation and GC events from Java Flight Recorder
 on a compatible Minecraft JVM. Allocation rate can rise while RSS stays flat.
 Native engine heap or frame profiles require that engine's profiling tools.
 
-At official module revision `48b8b65c`, the Minecraft Java, ARK Survival
-Ascended, Palworld, Project Zomboid and CS2 templates have no numeric tick
-or frame declarations. Minecraft Java's `seed` readout is display-only. Use
-a module version with an engine-verified command, response and unit before
-setting a performance alert for that game. A server tick-rate setting describes
-its configured target, while the time series must describe observed ticks.
+At official module revision `48b8b65c`, Minecraft Java, ARK Survival
+Ascended, Palworld, Project Zomboid and CS2 have no numeric tick or frame
+declarations. Minecraft Java's `seed` readout is display-only. The
+[Palworld module change](https://github.com/GameplanePanel/module/pull/60)
+declares instantaneous server FPS from the authenticated REST metrics
+endpoint. Its [engine fixture](https://github.com/GameplanePanel/module/blob/54e9a11a694c10b0dc2550391782b95bf1a883de/palworld/samples/metrics.json)
+records 59 FPS on Steam build 25247047. Its average FPS and frame time are
+separate fields. Select a module revision containing the declaration. Missing
+or malformed responses leave the numeric series absent. A server tick-rate
+setting describes a configured target, while this series describes observed
+frames.
 
 The Workloads dashboard shows scrape health, relay readiness/known state,
 supported game measurements, container pressure, restarts and Pending Pods.

From 1cd4039c30c5e99e78bc977af9f5740d6accc354 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Wed, 11 Nov 2026 01:11:19 -0500
Subject: [PATCH 4/6] fix(e2e): clear inherited probe allowances in the tenant
 monitoring drill

Signed-off-by: Baylem <zelathis@gmail.com>
---
 deploy/kind/monitoring.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh
index 4bc8e3dbf..0f5fd44f0 100644
--- a/deploy/kind/monitoring.sh
+++ b/deploy/kind/monitoring.sh
@@ -22,7 +22,7 @@ kubectl create namespace gameplane-monitoring-b --dry-run=client -o yaml | kubec
 helm --kube-context "$context" upgrade gameplane charts/gameplane --namespace gameplane-system --reuse-values \
   -f docs/examples/monitoring/gameplane-values.yaml \
   --set 'operator.gameNamespaces[0]=gameplane-monitoring-b' \
-  --set networkPolicies.tenantIsolation=true --set networkPolicies.probePorts=null \
+  --set networkPolicies.tenantIsolation=true --set-json 'networkPolicies.probePorts=[]' \
   --set networkPolicies.gameEgress.enabled=false --set networkPolicies.backupEgress.enabled=false \
   --wait --timeout 5m
 kubectl apply -f docs/examples/monitoring/prometheus.yaml

From ba9386f49ee159be46f76f5f731f1cf9e9f88fc6 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Wed, 11 Nov 2026 03:13:25 -0500
Subject: [PATCH 5/6] fix(e2e): scope tenant monitoring setup to its own suite

Signed-off-by: Baylem <zelathis@gmail.com>
---
 deploy/kind/monitoring.sh       |  7 +------
 test/e2e/monitoring_e2e_test.go | 14 ++++++++++++--
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh
index 0f5fd44f0..f5c2e4a4e 100644
--- a/deploy/kind/monitoring.sh
+++ b/deploy/kind/monitoring.sh
@@ -18,13 +18,8 @@ sed 's/namespace: default/namespace: monitoring/g' "$scratch/bundle.yaml" > "$sc
 kubectl apply --server-side -f "$scratch/monitoring.yaml"
 kubectl wait --for=condition=Established --timeout=120s crd/prometheuses.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com
 kubectl rollout status -n monitoring deployment/prometheus-operator --timeout=180s
-kubectl create namespace gameplane-monitoring-b --dry-run=client -o yaml | kubectl apply -f -
 helm --kube-context "$context" upgrade gameplane charts/gameplane --namespace gameplane-system --reuse-values \
-  -f docs/examples/monitoring/gameplane-values.yaml \
-  --set 'operator.gameNamespaces[0]=gameplane-monitoring-b' \
-  --set networkPolicies.tenantIsolation=true --set-json 'networkPolicies.probePorts=[]' \
-  --set networkPolicies.gameEgress.enabled=false --set networkPolicies.backupEgress.enabled=false \
-  --wait --timeout 5m
+  -f docs/examples/monitoring/gameplane-values.yaml --wait --timeout 5m
 kubectl apply -f docs/examples/monitoring/prometheus.yaml
 kubectl wait -n monitoring --for=create statefulset/prometheus-gameplane --timeout=120s
 kubectl rollout status -n monitoring statefulset/prometheus-gameplane --timeout=180s
diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go
index 1456d1945..36e1f3bfa 100644
--- a/test/e2e/monitoring_e2e_test.go
+++ b/test/e2e/monitoring_e2e_test.go
@@ -28,6 +28,18 @@ func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) {
 	if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil {
 		t.Fatalf("monitoring requires enforcing Cilium: %v", err)
 	}
+	stamp := time.Now().UnixNano()
+	otherTenant := fmt.Sprintf("gameplane-monitoring-%d", stamp)
+	namespaces := envInstance.K8s.CoreV1().Namespaces()
+	if _, err := namespaces.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherTenant}}, metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = namespaces.Delete(context.Background(), otherTenant, metav1.DeleteOptions{}) })
+	// The network-isolation suite shares the Prometheus bootstrap but first
+	// exercises legacy policies. Install strict per-tenant policy only here.
+	helmUpgradeReuseJSON(ctx, t, []string{"networkPolicies.probePorts=[]"},
+		"operator.gameNamespaces[0]="+otherTenant, "networkPolicies.tenantIsolation=true",
+		"networkPolicies.gameEgress.enabled=false", "networkPolicies.backupEgress.enabled=false")
 	port, stop := envInstance.PortForward(t, "monitoring", "svc/prometheus-operated", 9090)
 	defer stop()
 	client := &http.Client{Timeout: 10 * time.Second}
@@ -66,7 +78,6 @@ func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) {
 		})
 	}
 	envInstance.ApplyYAML(t, "frps.yaml")
-	stamp := time.Now().UnixNano()
 	name := fmt.Sprintf("monitoring-%d", stamp)
 	const gameCode = `from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
 import json,pathlib,threading
@@ -106,7 +117,6 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 		_ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{})
 	})
 	gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}}
-	const otherTenant = "gameplane-monitoring-b"
 	other := gs.DeepCopy()
 	other.SetNamespace(otherTenant)
 	if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Create(ctx, other, metav1.CreateOptions{}); err != nil {

From 633f77be7953d5a54afbbd72553e27bb4e921b32 Mon Sep 17 00:00:00 2001
From: Baylem <zelathis@gmail.com>
Date: Wed, 11 Nov 2026 03:16:54 -0500
Subject: [PATCH 6/6] fix(e2e): qualify monitored relays through their scoped
 FRP peer

Signed-off-by: Baylem <zelathis@gmail.com>
---
 test/e2e/monitoring_e2e_test.go | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go
index 36e1f3bfa..da7b2b5f5 100644
--- a/test/e2e/monitoring_e2e_test.go
+++ b/test/e2e/monitoring_e2e_test.go
@@ -77,7 +77,6 @@ func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) {
 			return err == nil && got == want, fmt.Sprintf("%s: got %s want %s (%v)", expression, got, want, err)
 		})
 	}
-	envInstance.ApplyYAML(t, "frps.yaml")
 	name := fmt.Sprintf("monitoring-%d", stamp)
 	const gameCode = `from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
 import json,pathlib,threading
@@ -90,7 +89,7 @@ class Handler(BaseHTTPRequestHandler):
     def log_message(self,*args): pass
     def do_GET(self):
         if self.path=='/allocate': heap.append(bytearray(16*1024*1024))
-        self.send_response(200); self.end_headers(); self.wfile.write(b'ok')
+        self.send_response(200); self.end_headers(); self.wfile.write(b'reachable\n')
     def do_POST(self):
         command=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0')))).get('command','')
         if command=='stats':
@@ -116,7 +115,11 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 	t.Cleanup(func() {
 		_ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{})
 	})
-	gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}}
+	gs := &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer",
+		"metadata": map[string]any{"name": name, "namespace": ocGamesNS},
+		"spec":     map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP"}},
+	}}
 	other := gs.DeepCopy()
 	other.SetNamespace(otherTenant)
 	if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Create(ctx, other, metav1.CreateOptions{}); err != nil {
@@ -125,21 +128,13 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever()
 	t.Cleanup(func() {
 		_ = envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Delete(context.Background(), name, metav1.DeleteOptions{})
 	})
-	gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{})
-	if err != nil {
+	if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{}); err != nil {
 		t.Fatal(err)
 	}
 	t.Cleanup(func() {
 		_ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Delete(context.Background(), name, metav1.DeleteOptions{})
 	})
-	secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-auth", Namespace: ocGamesNS, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: name, UID: gs.GetUID()}}}, StringData: map[string]string{"token": ocFrpsToken}}
-	if _, err := envInstance.K8s.CoreV1().Secrets(ocGamesNS).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
-		t.Fatal(err)
-	}
-	patch, _ := json.Marshal(map[string]any{"spec": map[string]any{"networking": map[string]any{"tunnel": map[string]any{"enabled": true, "credentialsSecretRef": map[string]any{"name": secret.Name}}}}})
-	if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
-		t.Fatal(err)
-	}
+	enableIsolatedFRP(t, ocGamesNS, name)
 	selector := fmt.Sprintf(`namespace="gameplane-games",server="%s"`, name)
 	waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1")
 	otherSelector := fmt.Sprintf(`namespace="%s",server="%s"`, otherTenant, name)