diff --git a/changelog/unreleased/908-tenant-monitoring.md b/changelog/unreleased/908-tenant-monitoring.md new file mode 100644 index 000000000..c550bca14 --- /dev/null +++ b/changelog/unreleased/908-tenant-monitoring.md @@ -0,0 +1,4 @@ +# Monitoring across managed tenants + +Prometheus discovers game workloads in managed namespaces and distinguishes missing targets +from failed scrapes. Dashboards show process RSS and container working set separately. [PR #908](https://github.com/GameplanePanel/Gameplane/pull/908) diff --git a/charts/gameplane/dashboards/gameplane-workloads.json b/charts/gameplane/dashboards/gameplane-workloads.json index f0a978b26..cc0b43522 100644 --- a/charts/gameplane/dashboards/gameplane-workloads.json +++ b/charts/gameplane/dashboards/gameplane-workloads.json @@ -340,6 +340,46 @@ "legendFormat": "{{namespace}} / {{pod}} {{metric}}" } ] + }, + { + "id": 11, + "title": "Relay drops and rejected flows", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "ops", "noValue": "Unknown", "custom": { "spanNulls": false } } }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod,protocol) (rate(gameplane_tunnel_flow_rejections_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "{{namespace}} / {{pod}} {{protocol}} rejections" + }, + { + "refId": "B", + "expr": "max by (namespace,pod) (rate(gameplane_tunnel_udp_queue_drops_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "{{namespace}} / {{pod}} UDP queue drops" + } + ] + }, + { + "id": 12, + "title": "Game process RSS and container working set", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "fieldConfig": { "defaults": { "unit": "bytes", "noValue": "Unknown", "custom": { "spanNulls": false } } }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (gameplane_agent_memory_bytes{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} process RSS" + }, + { + "refId": "B", + "expr": "max by (namespace,pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"game\"})", + "legendFormat": "{{namespace}} / {{pod}} working set" + } + ] } ] } diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml index 98b383683..181b0d16f 100644 --- a/charts/gameplane/templates/monitoring-rules.yaml +++ b/charts/gameplane/templates/monitoring-rules.yaml @@ -1,4 +1,5 @@ {{- if and .Values.prometheusRules.enabled (eq (include "gameplane.operatorEnabled" .) "true") }} +{{- $namespaces := join "|" (include "gameplane.operatorGameNamespaces" . | fromJsonArray) }} apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: @@ -13,59 +14,79 @@ spec: groups: - name: gameplane.workloads rules: + - alert: GameplaneScrapeTargetMissing + expr: | + ( + max by (namespace, pod, gameplane_component) ( + label_replace(kube_pod_container_status_running{namespace=~"{{ $namespaces }}",container=~"agent|tunnel"} == 1, + "gameplane_component", "$1", "container", "(agent|tunnel)") + ) + and on (namespace, pod) kube_pod_labels{namespace=~"{{ $namespaces }}",label_app_kubernetes_io_name=~"gameplane-game|gameplane-tunnel"} + ) + unless on (namespace, pod, gameplane_component) up{namespace=~"{{ $namespaces }}",gameplane_component=~"agent|tunnel"} + for: 5m + labels: { severity: warning } + annotations: { summary: "Running Gameplane workload is missing from scrape discovery" } - alert: GameplaneScrapeUnavailable - expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace="{{ .Values.gamesNamespace }}"}) == 0 + expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace=~"{{ $namespaces }}"}) == 0 for: 5m labels: { severity: warning } annotations: { summary: "Gameplane metrics target cannot be scraped" } - alert: GameplaneRelayUnavailable - expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 1) + expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 1) for: 2m labels: { severity: warning } annotations: { summary: "Gameplane relay connection or forwards are unavailable" } - alert: GameplaneRelayHealthUnknown - expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace="{{ .Values.gamesNamespace }}"}) == 1) + expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace=~"{{ $namespaces }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace=~"{{ $namespaces }}"}) == 1) for: 5m labels: { severity: info } annotations: { summary: "Gameplane provider cannot confirm live relay health" } + - alert: GameplaneRelaySaturated + expr: | + max by (namespace, pod) (increase(gameplane_tunnel_flow_rejections_total{namespace=~"{{ $namespaces }}"}[5m])) > 0 + or + max by (namespace, pod) (increase(gameplane_tunnel_udp_queue_drops_total{namespace=~"{{ $namespaces }}"}[5m])) > 0 + labels: { severity: warning } + annotations: { summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets" } - alert: GameplanePerformanceUnknown - expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace="{{ .Values.gamesNamespace }}"}) == 0 + expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace=~"{{ $namespaces }}"}) == 0 for: 5m labels: { severity: warning } annotations: { summary: "Declared game performance measurement is unavailable" } - alert: GameplanePerformanceDegraded expr: | - (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"}) < on (namespace, pod, metric, signal, unit) - max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="below"})) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="below"})) or - (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace=~"{{ $namespaces }}"}) > on (namespace, pod, metric, signal, unit) - max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="above"})) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace=~"{{ $namespaces }}",direction="above"})) for: 5m labels: { severity: warning } annotations: { summary: "Game performance crossed its module-declared threshold" } - alert: GameplaneCPUThrottled - expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m])) > 0.25 + expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace=~"{{ $namespaces }}",container="game"}[5m])) > 0.25 for: 10m labels: { severity: warning } annotations: { summary: "Game container CPU is throttled in more than 25% of periods" } - alert: GameplaneMemoryPressure - expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} / (container_spec_memory_limit_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} > 0)) > 0.9 + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace=~"{{ $namespaces }}",container="game"} / (container_spec_memory_limit_bytes{namespace=~"{{ $namespaces }}",container="game"} > 0)) > 0.9 for: 10m labels: { severity: warning } annotations: { summary: "Game container working set exceeds 90% of its memory limit" } - alert: GameplaneContainerOOM - expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="{{ .Values.gamesNamespace }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace="{{ .Values.gamesNamespace }}",reason="OOMKilled"}) == 1) + expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace=~"{{ $namespaces }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace=~"{{ $namespaces }}",reason="OOMKilled"}) == 1) labels: { severity: warning } annotations: { summary: "Gameplane container restarted after an OOM kill" } - alert: GameplanePodPending - expr: max by (namespace, pod) (kube_pod_status_phase{namespace="{{ .Values.gamesNamespace }}",phase="Pending"}) == 1 + expr: max by (namespace, pod) (kube_pod_status_phase{namespace=~"{{ $namespaces }}",phase="Pending"}) == 1 for: 10m labels: { severity: warning } annotations: { summary: "Gameplane Pod has remained Pending for ten minutes" } - alert: GameplaneNodePressure - expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace="{{ .Values.gamesNamespace }}"}) + expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace=~"{{ $namespaces }}"}) for: 5m labels: { severity: warning } annotations: { summary: "A node hosting Gameplane Pods reports resource pressure" } diff --git a/charts/gameplane/templates/servicemonitors.yaml b/charts/gameplane/templates/servicemonitors.yaml index 2b5b6834b..2ae0b26e4 100644 --- a/charts/gameplane/templates/servicemonitors.yaml +++ b/charts/gameplane/templates/servicemonitors.yaml @@ -101,6 +101,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + namespaceSelector: + matchNames: {{ include "gameplane.operatorGameNamespaces" . }} selector: matchLabels: { app.kubernetes.io/name: gameplane-game } podMetricsEndpoints: @@ -124,6 +126,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + namespaceSelector: + matchNames: {{ include "gameplane.operatorGameNamespaces" . }} selector: matchLabels: { app.kubernetes.io/name: gameplane-tunnel } podMetricsEndpoints: diff --git a/charts/gameplane/tests/monitoring-rules.sh b/charts/gameplane/tests/monitoring-rules.sh index 6e39efd8e..ae876184e 100644 --- a/charts/gameplane/tests/monitoring-rules.sh +++ b/charts/gameplane/tests/monitoring-rules.sh @@ -4,14 +4,20 @@ set -euo pipefail cd "$(dirname "$0")/../../.." scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT -helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml > "$scratch/rendered.yaml" +helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml \ + --set 'operator.gameNamespaces={tenant-b,gameplane-games}' > "$scratch/rendered.yaml" python3 - "$scratch" <<'PY' import pathlib,sys,yaml root=pathlib.Path(sys.argv[1]) groups=[] +monitors={} for document in yaml.safe_load_all((root/'rendered.yaml').read_text()): if document and document.get('kind')=='PrometheusRule': groups.extend(document['spec']['groups']) + if document and document.get('kind')=='PodMonitor': + monitors[document['metadata']['name']]=document['spec']['namespaceSelector']['matchNames'] +expected=['gameplane-games','tenant-b'] +assert monitors=={'gameplane-agent':expected,'gameplane-tunnel':expected}, monitors (root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups})) PY cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml" diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml index 2d285dedf..d272e8a6f 100644 --- a/charts/gameplane/tests/monitoring-rules.test.yaml +++ b/charts/gameplane/tests/monitoring-rules.test.yaml @@ -1,6 +1,54 @@ rule_files: [rules.yaml] evaluation_interval: 1m tests: + - interval: 1m + input_series: + - series: 'gameplane_tunnel_flow_rejections_total{namespace="gameplane-games",pod="busy",protocol="tcp"}' + values: '0x2 1x10' + - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="tenant-b",pod="backlogged"}' + values: '0x2 1x10' + - series: 'gameplane_tunnel_udp_queue_drops_total{namespace="unmanaged",pod="outside"}' + values: '0+10x12' + - series: 'gameplane_tunnel_forward_metrics_supported{namespace="tenant-b",pod="opaque"}' + values: '0x12' + alert_rule_test: + - eval_time: 4m + alertname: GameplaneRelaySaturated + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: busy, severity: warning} + exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"} + - exp_labels: {namespace: tenant-b, pod: backlogged, severity: warning} + exp_annotations: {summary: "Relay rejected traffic at its flow limit or dropped queued UDP packets"} + - eval_time: 10m + alertname: GameplaneRelaySaturated + exp_alerts: [] + - interval: 1m + input_series: + - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="sleepy-0",container="agent"}' + values: '1x12' + - series: 'kube_pod_labels{namespace="tenant-b",pod="sleepy-0",label_app_kubernetes_io_name="gameplane-game"}' + values: '1x12' + - series: 'kube_pod_container_status_running{namespace="tenant-b",pod="relay",container="tunnel"}' + values: '1x12' + - series: 'kube_pod_labels{namespace="tenant-b",pod="relay",label_app_kubernetes_io_name="gameplane-tunnel"}' + values: '1x12' + - series: 'up{namespace="tenant-b",pod="relay",gameplane_component="tunnel"}' + values: '0x12' + - series: 'kube_pod_container_status_running{namespace="unmanaged",pod="hidden",container="agent"}' + values: '1x12' + - series: 'kube_pod_labels{namespace="unmanaged",pod="hidden",label_app_kubernetes_io_name="gameplane-game"}' + values: '1x12' + alert_rule_test: + - eval_time: 8m + alertname: GameplaneScrapeTargetMissing + exp_alerts: + - exp_labels: {namespace: tenant-b, pod: sleepy-0, gameplane_component: agent, severity: warning} + exp_annotations: {summary: "Running Gameplane workload is missing from scrape discovery"} + - eval_time: 8m + alertname: GameplaneScrapeUnavailable + exp_alerts: + - exp_labels: {namespace: tenant-b, pod: relay, gameplane_component: tunnel, severity: warning} + exp_annotations: {summary: "Gameplane metrics target cannot be scraped"} - interval: 1m input_series: - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds"}' diff --git a/docs/monitoring.md b/docs/monitoring.md index 6f6245947..e11e2896c 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -28,6 +28,26 @@ Check Prometheus **Status → Targets** for agent and tunnel targets with `up=1` then query `gameplane_agent_players_online` and `gameplane_tunnel_ready`. Unknown player counts are absent even when scraping works. +The agent and tunnel PodMonitors live in the primary games namespace. Their +target lists include that namespace and every entry in `operator.gameNamespaces`. +Ensure Prometheus honors PodMonitor namespace selectors and can list and watch +pods in each target namespace. Provision the same approved scraper ingress in +additional namespaces that enforce default deny, and allow Prometheus egress. +Workload alerts select the same explicit list of namespaces. + +For installations using `networkPolicies.tenantIsolation: true`, the chart +installs the approved scraper ingress in every managed game namespace. Tenant +operators should query a tenant-scoped Prometheus endpoint or a datasource +that enforces their namespace. A Grafana variable filters a view and grants no +authorization. Keep the fleet-wide Prometheus credentials with operators. + +`GameplaneScrapeUnavailable` fires when a discovered endpoint stays down. +`GameplaneScrapeTargetMissing` compares discovered targets with running +agent/tunnel containers on Gameplane-labeled pods. Enable kube-state-metrics +pod label collection for `app.kubernetes.io/name` (for example +`--metric-labels-allowlist=pods=[app.kubernetes.io/name]`) and scrape its +container-running and pod-label metrics to enable that comparison. + With the operator and its NetworkPolicies enabled, the chart requires a nonempty `serviceMonitors.scrapeNamespaceSelector`. Missing configuration fails with a message naming the setting. If another system owns the policies, set @@ -135,16 +155,41 @@ staleness limit. See [relay-health.md](relay-health.md). Tailscale, frp and Playit all use the same metrics path. Metrics contain no credentials, login URLs, proxy names or provider error messages. +`GameplaneRelaySaturated` fires on an increase in a supervisor flow-rejection +or UDP queue-drop counter over five minutes. The dashboard shows both rates +by namespace and Pod. The embedded Tailscale forwarder supplies these counters. +For frp and Playit the support gauge is zero and the counters are absent. +Use provider-side measurements for those paths. + Container CPU throttling and working-set pressure require kubelet/cAdvisor scrapes. OOM/restart, Pending Pod and node pressure rules require kube-state-metrics. Install/configure those collectors (kube-prometheus-stack normally provides them); the minimal Prometheus example demonstrates Gameplane discovery and does not deploy them or Grafana. A missing upstream collector leaves those series absent; it does not mean zero pressure. Agent process RSS is not total -container memory. These rules filter the games namespace and use `max` to +container memory. These rules filter the explicitly managed game namespaces and use `max` to deduplicate replicated sources. The existing fleet rules likewise collapse duplicate operator replicas instead of adding their cache-derived counts. +In operator-managed games the RSS series sums the game processes in the shared +PID namespace. cAdvisor reports the game container's working set independently. +Distinguish sustained growth from allocator churn by correlating both with an +engine profiler, such as JVM allocation and GC events from Java Flight Recorder +on a compatible Minecraft JVM. Allocation rate can rise while RSS stays flat. +Native engine heap or frame profiles require that engine's profiling tools. + +At official module revision `48b8b65c`, Minecraft Java, ARK Survival +Ascended, Palworld, Project Zomboid and CS2 have no numeric tick or frame +declarations. Minecraft Java's `seed` readout is display-only. The +[Palworld module change](https://github.com/GameplanePanel/module/pull/60) +declares instantaneous server FPS from the authenticated REST metrics +endpoint. Its [engine fixture](https://github.com/GameplanePanel/module/blob/54e9a11a694c10b0dc2550391782b95bf1a883de/palworld/samples/metrics.json) +records 59 FPS on Steam build 25247047. Its average FPS and frame time are +separate fields. Select a module revision containing the declaration. Missing +or malformed responses leave the numeric series absent. A server tick-rate +setting describes a configured target, while this series describes observed +frames. + The Workloads dashboard shows scrape health, relay readiness/known state, supported game measurements, container pressure, restarts and Pending Pods. Missing measurements remain gaps. CI renders and evaluates the shipped alert diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go index 00bdb02ae..da7b2b5f5 100644 --- a/test/e2e/monitoring_e2e_test.go +++ b/test/e2e/monitoring_e2e_test.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ) @@ -27,6 +28,18 @@ func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) { if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { t.Fatalf("monitoring requires enforcing Cilium: %v", err) } + stamp := time.Now().UnixNano() + otherTenant := fmt.Sprintf("gameplane-monitoring-%d", stamp) + namespaces := envInstance.K8s.CoreV1().Namespaces() + if _, err := namespaces.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: otherTenant}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = namespaces.Delete(context.Background(), otherTenant, metav1.DeleteOptions{}) }) + // The network-isolation suite shares the Prometheus bootstrap but first + // exercises legacy policies. Install strict per-tenant policy only here. + helmUpgradeReuseJSON(ctx, t, []string{"networkPolicies.probePorts=[]"}, + "operator.gameNamespaces[0]="+otherTenant, "networkPolicies.tenantIsolation=true", + "networkPolicies.gameEgress.enabled=false", "networkPolicies.backupEgress.enabled=false") port, stop := envInstance.PortForward(t, "monitoring", "svc/prometheus-operated", 9090) defer stop() client := &http.Client{Timeout: 10 * time.Second} @@ -64,25 +77,25 @@ func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) { return err == nil && got == want, fmt.Sprintf("%s: got %s want %s (%v)", expression, got, want, err) }) } - envInstance.ApplyYAML(t, "frps.yaml") - stamp := time.Now().UnixNano() name := fmt.Sprintf("monitoring-%d", stamp) const gameCode = `from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer import json,pathlib,threading value=pathlib.Path('/tmp/performance') count=pathlib.Path('/tmp/performance-queries') value.write_text('20') +heap=[] count.write_text('0') class Handler(BaseHTTPRequestHandler): def log_message(self,*args): pass def do_GET(self): - self.send_response(200); self.end_headers(); self.wfile.write(b'ok') + if self.path=='/allocate': heap.append(bytearray(16*1024*1024)) + self.send_response(200); self.end_headers(); self.wfile.write(b'reachable\n') def do_POST(self): command=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0')))).get('command','') if command=='stats': count.write_text(str(int(count.read_text())+1)) output='TPS: '+value.read_text()+' Tick: 50' - else: output='0 players online' + else: output='unavailable' if pathlib.Path('/tmp/players-unknown').exists() else '0 players online' self.send_response(200); self.end_headers(); self.wfile.write(json.dumps({'output':output}).encode()) threading.Thread(target=ThreadingHTTPServer(('127.0.0.1',25575),Handler).serve_forever,daemon=True).start() ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever() @@ -102,28 +115,57 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever() t.Cleanup(func() { _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) }) - gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}} - gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{}) - if err != nil { + gs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", + "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, + "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP"}}, + }} + other := gs.DeepCopy() + other.SetNamespace(otherTenant) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Create(ctx, other, metav1.CreateOptions{}); err != nil { t.Fatal(err) } t.Cleanup(func() { - _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Delete(context.Background(), name, metav1.DeleteOptions{}) + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(otherTenant).Delete(context.Background(), name, metav1.DeleteOptions{}) }) - secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-auth", Namespace: ocGamesNS, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: name, UID: gs.GetUID()}}}, StringData: map[string]string{"token": ocFrpsToken}} - if _, err := envInstance.K8s.CoreV1().Secrets(ocGamesNS).Create(ctx, secret, metav1.CreateOptions{}); err != nil { - t.Fatal(err) - } - patch, _ := json.Marshal(map[string]any{"spec": map[string]any{"networking": map[string]any{"tunnel": map[string]any{"enabled": true, "credentialsSecretRef": map[string]any{"name": secret.Name}}}}}) - if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{}); err != nil { t.Fatal(err) } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + enableIsolatedFRP(t, ocGamesNS, name) selector := fmt.Sprintf(`namespace="gameplane-games",server="%s"`, name) waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + otherSelector := fmt.Sprintf(`namespace="%s",server="%s"`, otherTenant, name) + waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1") + // Invalidate the player response on one of two identically named games. + waitQuery(`max(gameplane_agent_players_online{`+otherSelector+`})`, "0") + if out, err := envInstance.Kubectl(ctx, "exec", "-n", otherTenant, name+"-0", "-c", "game", "--", "touch", "/tmp/players-unknown"); err != nil { + t.Fatalf("invalidate player count: %v %s", err, out) + } + waitQuery(`absent(gameplane_agent_players_online{`+otherSelector+`})`, "1") + waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1") + waitQuery(`max(gameplane_agent_players_online{`+selector+`})`, "0") + waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+otherSelector+`})`, "20") waitQuery(`max(up{gameplane_component="tunnel",`+selector+`})`, "1") waitQuery(`max(gameplane_tunnel_ready{`+selector+`})`, "1") waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, "20") waitQuery(`max(gameplane_agent_performance_value{metric="tick",`+selector+`})`, "0.05") + rssQuery := `max(gameplane_agent_memory_bytes{` + selector + `})` + waitQuery(rssQuery+` > bool 0`, "1") + baseline, err := query(rssQuery) + if err != nil { + t.Fatal(err) + } + rss, err := strconv.ParseFloat(baseline, 64) + if err != nil { + t.Fatal(err) + } + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `import urllib.request; urllib.request.urlopen('http://127.0.0.1:25566/allocate').read()`); err != nil { + t.Fatalf("allocate memory in game process: %v %s", err, out) + } + waitQuery(fmt.Sprintf(`%s > bool %.0f`, rssQuery, rss+8*1024*1024), "1") waitQuery(`count(gameplane_agent_performance_value{metric="display",`+selector+`}) or vector(0)`, "0") readCount := func() int { out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; print(Path('/tmp/performance-queries').read_text())`) @@ -140,6 +182,39 @@ ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever() if err != nil { t.Fatal(err) } + otherProbe := name + "-other" + monitoringProbe(t, otherTenant, otherProbe) + // Remove one tenant from PodMonitor discovery while its pod remains ready. + monitors := envInstance.Dyn.Resource(schema.GroupVersionResource{Group: "monitoring.coreos.com", Version: "v1", Resource: "podmonitors"}).Namespace(ocGamesNS) + monitor, err := monitors.Get(ctx, "gameplane-agent", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + names, _, err := unstructured.NestedStringSlice(monitor.Object, "spec", "namespaceSelector", "matchNames") + if err != nil { + t.Fatal(err) + } + patchNamespaces := func(ctx context.Context, namespaces []string) error { + body, err := json.Marshal(map[string]any{"spec": map[string]any{"namespaceSelector": map[string]any{"matchNames": namespaces}}}) + if err != nil { + return err + } + _, err = monitors.Patch(ctx, "gameplane-agent", types.MergePatchType, body, metav1.PatchOptions{}) + return err + } + t.Cleanup(func() { _ = patchNamespaces(context.Background(), names) }) + if err := patchNamespaces(ctx, []string{ocGamesNS}); err != nil { + t.Fatal(err) + } + waitQuery(`absent(up{gameplane_component="agent",`+otherSelector+`})`, "1") + if ready, err := envInstance.PodIsReady(ctx, otherTenant, name+"-0"); err != nil || !ready { + t.Fatalf("missing target pod ready=%t err=%v", ready, err) + } + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + if err := patchNamespaces(ctx, names); err != nil { + t.Fatal(err) + } + waitQuery(`max(up{gameplane_component="agent",`+otherSelector+`})`, "1") approvedProbe := name + "-approved" monitoringProbe(t, "monitoring", approvedProbe) before := readCount() @@ -195,9 +270,11 @@ try: except (OSError,TimeoutError): print('DENIED') ` - out, err := envInstance.Kubectl(ctx, "exec", "-n", ocNS, probeName, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics") - if err != nil || strings.TrimSpace(out) != "DENIED" { - t.Fatalf("unapproved namespace reached metrics, or probe failed: %v %s", err, out) + for namespace, probe := range map[string]string{ocNS: probeName, otherTenant: otherProbe} { + out, err := envInstance.Kubectl(ctx, "exec", "-n", namespace, probe, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics") + if err != nil || strings.TrimSpace(out) != "DENIED" { + t.Fatalf("unapproved namespace %s reached metrics, or probe failed: %v %s", namespace, err, out) + } } } diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 510102c67..efb3e0bf3 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -57,7 +57,7 @@ stop, status reset/leader recovery, exact request binding, snapshot integrity, complete pinned render inputs and source-template preservation for CSI restore. - Keep the lint workflow, devcontainer and `lint-gate-verify.sh` on the exact `v2.14.0` linter pin, whose importer supports Go 1.27.3 export data. The verifier's valid fixtures use the same pin; regression fixtures reject `latest` and a stale fixed version in any lint step. -- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It discovers real agent/tunnel targets through the documented chart values, verifies cached numeric samples, valid zero, invalid-value omission, no scrape-driven commands, policy revocation/recovery and refusal from an unapproved namespace. It performs no API login and does not opt in external modules. +- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It runs identically named games in two tenant namespaces, removes and restores one discovery target, revokes and restores scrape ingress, and verifies that a tenant and a management Pod cannot read another tenant's metrics. It checks zero versus unknown player counts, game-process memory growth, numeric tick samples and invalid-value omission. Repeated scrapes must use the interval cache. The game fixture implements deterministic RCON responses and requires no external module or API login. - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced.