From c20343c160cc5b6c83a4edc3550d089297520518 Mon Sep 17 00:00:00 2001 From: wayne Date: Mon, 9 Nov 2026 21:17:57 +0800 Subject: [PATCH 1/2] feat(deploy): add an opt-in production control capacity profile Provide a single-active Coordinator, four API replicas with required host separation, and persistent single-instance Redis as a sizing starting point. Keep small-installation defaults and existing worker/runtime behavior intact. Allow explicit Redis control-pool placement. Document SSD selection, the four-to-eight API capacity budget, safe existing-PVC expansion and rollback, and qualification required before claiming target sandbox or connection scale. Validate core and umbrella rendering, default compatibility and site overrides. Signed-off-by: wayne --- AGENTS.md | 7 + deploy/README.md | 3 + .../charts/core/templates/adx/redis.yaml | 9 ++ .../core/tests/test_production_capacity.py | 96 ++++++++++++ .../charts/core/values-production.yaml | 52 +++++++ deploy/akernel/charts/core/values.yaml | 4 + deploy/docs/control-plane-capacity.md | 140 ++++++++++++++++++ 7 files changed, 311 insertions(+) create mode 100644 deploy/akernel/charts/core/tests/test_production_capacity.py create mode 100644 deploy/akernel/charts/core/values-production.yaml create mode 100644 deploy/docs/control-plane-capacity.md diff --git a/AGENTS.md b/AGENTS.md index decf6322..3734581f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,6 +288,13 @@ environment overrides take precedence. Preserve this handoff when changing node deployment: otherwise the service can fall back to standalone configuration even though the Pod declares a node configuration. +Large deployments may opt into the core chart's `values-production.yaml`. +The [control-plane capacity guide](deploy/docs/control-plane-capacity.md) owns +resource budgets, strict API host separation, SSD selection and existing-PVC +migration. These are starting allocations, not a 100,000-sandbox capacity claim. +Keep Redis placement optional and do not apply a capacity profile to a live +release without reviewing current values, data and rollout requirements. + The managed Kubernetes Redis uses its own Helm-generated authentication Secret, preserved by live lookup during upgrades. Its NetworkPolicy is additional isolation and requires CNI enforcement. Coordinator and Ingress/API Server run in separate diff --git a/deploy/README.md b/deploy/README.md index c5c1eb3f..7d640bd2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -476,6 +476,9 @@ The default chart runs ADX Ingress and API Server in one `akernel-adx-ingress-api` Pod, with 2 CPU requested and an 8 CPU limit. Override `core.adx.ingressApi.resources` for your node shape and workload. Measure capacity with the [Create benchmark](../sdk/python/benchmarks/create_throughput/README.md). +For larger installations, use the opt-in [production control-plane capacity +profile](docs/control-plane-capacity.md), including placement, SSD selection, +capacity qualification and existing-PVC migration requirements. The `monitor` subchart remains optional and contains Prometheus, Grafana, Loki, and Tempo. Kubernetes nodes must support privileged Pods and the runtime diff --git a/deploy/akernel/charts/core/templates/adx/redis.yaml b/deploy/akernel/charts/core/templates/adx/redis.yaml index 5fb81773..a44aaf6e 100644 --- a/deploy/akernel/charts/core/templates/adx/redis.yaml +++ b/deploy/akernel/charts/core/templates/adx/redis.yaml @@ -42,6 +42,15 @@ spec: labels: app: akernel-adx-redis spec: + {{- with .Values.adx.redis.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.adx.redis.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.adx.redis.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/deploy/akernel/charts/core/tests/test_production_capacity.py b/deploy/akernel/charts/core/tests/test_production_capacity.py new file mode 100644 index 00000000..bb04937f --- /dev/null +++ b/deploy/akernel/charts/core/tests/test_production_capacity.py @@ -0,0 +1,96 @@ +"""Production profile placement and compatibility contracts; no scale claim.""" +import copy +import subprocess +import unittest +from pathlib import Path + +import yaml + +CHART = Path(__file__).resolve().parents[1] +PROFILE = yaml.safe_load((CHART / "values-production.yaml").read_text()) + + +def render(values, umbrella=False): + chart = CHART.parents[1] if umbrella else CHART + result = subprocess.run( + ["helm", "template", "akernel", str(chart), "-n", "akernel", "-f", "-"], + input=yaml.safe_dump(values), text=True, capture_output=True, check=True, + ) + return {(x["kind"], x["metadata"]["name"]): x + for x in yaml.safe_load_all(result.stdout) if x} + + +class ProductionCapacityTests(unittest.TestCase): + def test_profile_keeps_single_active_coordinator_and_persistent_redis(self): + objects = render(PROFILE) + coordinator = objects["Deployment", "akernel-adx-coordinator"]["spec"] + self.assertEqual(coordinator["replicas"], 1) + self.assertEqual(coordinator["strategy"]["type"], "Recreate") + redis = objects["StatefulSet", "akernel-adx-redis"]["spec"] + self.assertEqual(redis["replicas"], 1) + container = redis["template"]["spec"]["containers"][0] + self.assertEqual(container["args"][container["args"].index("--appendonly") + 1], "yes") + self.assertEqual(container["args"][container["args"].index("--appendfsync") + 1], "everysec") + self.assertEqual(container["env"][0]["valueFrom"]["secretKeyRef"]["name"], "akernel-adx-redis-auth") + claim = redis["volumeClaimTemplates"][0]["spec"] + self.assertEqual(claim["resources"]["requests"]["storage"], "200Gi") + self.assertEqual(claim["accessModes"], ["ReadWriteOnce"]) + self.assertNotIn("storageClassName", claim) # operator must select qualified SSD + + def test_api_host_separation_scales_with_replica_count_and_pdb(self): + for replicas in (4, 8): + values = copy.deepcopy(PROFILE) + values["adx"]["ingressApi"]["replicas"] = replicas + objects = render(values) + api = objects["Deployment", "akernel-adx-ingress-api"]["spec"] + self.assertEqual(api["replicas"], replicas) + pod = api["template"]["spec"] + rule = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0] + self.assertEqual(rule["topologyKey"], "kubernetes.io/hostname") + self.assertEqual(rule["labelSelector"]["matchLabels"], api["selector"]["matchLabels"]) + self.assertEqual(pod["topologySpreadConstraints"][0]["whenUnsatisfiable"], "ScheduleAnyway") + self.assertEqual(api["strategy"]["rollingUpdate"], {"maxSurge": 0, "maxUnavailable": 1}) + self.assertEqual(objects["PodDisruptionBudget", "akernel-adx-ingress-api"]["spec"]["minAvailable"], replicas - 1) + + def test_profile_changes_only_capacity_and_api_placement(self): + before, after = render({}), render(PROFILE) + allowed = {("Deployment", "akernel-adx-coordinator"), + ("Deployment", "akernel-adx-ingress-api"), + ("StatefulSet", "akernel-adx-redis"), + ("PodDisruptionBudget", "akernel-adx-ingress-api"), + ("Secret", "akernel-adx-redis-auth")} # offline random password + self.assertEqual(after.keys() - before.keys(), {("PodDisruptionBudget", "akernel-adx-ingress-api")}) + for key in before.keys() - allowed: + self.assertEqual(before[key], after[key], key) + for kind, name, role in (("Deployment", "akernel-adx-coordinator", "coordinator"), + ("Deployment", "akernel-adx-ingress-api", "ingressApi"), + ("StatefulSet", "akernel-adx-redis", "redis")): + pod = after[kind, name]["spec"]["template"]["spec"] + self.assertEqual(pod["containers"][0]["resources"], PROFILE["adx"][role]["resources"]) + + def test_redis_placement_and_storage_site_overrides(self): + values = copy.deepcopy(PROFILE) + placement = {"nodeSelector": {"pool": "control"}, + "tolerations": [{"key": "control", "operator": "Exists", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": []}}} + values["adx"]["redis"].update(placement) + values["adx"]["redis"]["persistence"].update(storageClassName="qualified-ssd", size="300Gi") + spec = render(values)["StatefulSet", "akernel-adx-redis"]["spec"] + for key, value in placement.items(): + self.assertEqual(spec["template"]["spec"][key], value) + self.assertEqual(spec["volumeClaimTemplates"][0]["spec"]["storageClassName"], "qualified-ssd") + self.assertEqual(spec["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], "300Gi") + baseline = render({})["StatefulSet", "akernel-adx-redis"]["spec"]["template"]["spec"] + for key in placement: + self.assertNotIn(key, baseline) + + def test_umbrella_uses_explicit_core_wrapper_and_retains_light_defaults(self): + objects = render({"core": PROFILE, "monitor": {"enabled": False}}, umbrella=True) + self.assertEqual(objects["Deployment", "akernel-adx-ingress-api"]["spec"]["replicas"], 4) + default = render({}) + self.assertEqual(default["Deployment", "akernel-adx-ingress-api"]["spec"]["replicas"], 1) + self.assertEqual(default["StatefulSet", "akernel-adx-redis"]["spec"]["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], "20Gi") + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/akernel/charts/core/values-production.yaml b/deploy/akernel/charts/core/values-production.yaml new file mode 100644 index 00000000..582299de --- /dev/null +++ b/deploy/akernel/charts/core/values-production.yaml @@ -0,0 +1,52 @@ +# Opt-in core-chart capacity starting point, NOT a measured capacity guarantee. +# Read deploy/docs/control-plane-capacity.md before installing or upgrading. +# Supply a qualified SSD StorageClass and placement overrides for your cluster. +adx: + coordinator: + replicas: 1 + resources: + requests: + cpu: "4" + memory: 16Gi + limits: + cpu: "8" + memory: 32Gi + ingressApi: + replicas: 4 + resources: + requests: + cpu: "4" + memory: 8Gi + limits: + cpu: "8" + memory: 16Gi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: akernel-adx-ingress-api + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: akernel-adx-ingress-api + redis: + mode: managed + appendfsync: everysec + persistence: + enabled: true + # Must override with a qualified SSD-backed StorageClass; no portable name. + storageClassName: "" + size: 200Gi + accessModes: [ReadWriteOnce] + resources: + requests: + cpu: "4" + memory: 32Gi + limits: + cpu: "8" + memory: 64Gi diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index eab9c69d..aad434dd 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -76,7 +76,11 @@ adx: tls: existingSecret: akernel-adx-tls redis: + # Optional production capacity baseline: values-production.yaml. mode: managed + nodeSelector: {} + tolerations: [] + affinity: {} appendfsync: everysec external: existingSecret: "" diff --git a/deploy/docs/control-plane-capacity.md b/deploy/docs/control-plane-capacity.md new file mode 100644 index 00000000..04de9eaf --- /dev/null +++ b/deploy/docs/control-plane-capacity.md @@ -0,0 +1,140 @@ +# Production control-plane capacity profile + +The core chart's [values-production.yaml](../akernel/charts/core/values-production.yaml) +is an opt-in starting allocation for installations targeting roughly 100,000 +running sandboxes and 10,000 active connections. These targets are **not measured +capacity guarantees**. The default values remain suitable for smaller installations; +this profile does not change standalone deployment or provision cloud nodes. + +Ingress and API Server run together in each API Pod. Both protocols continue to +share one control plane and worker pool. This profile neither enables E2B nor +changes TLS, credentials, public Services, worker resources or checkpoint storage. + +## Allocation and placement + +Each row describes the main container; budget separately for collectors, system +services, monitoring, image distribution and workload nodes. + +| Component | Replicas | CPU request / limit | Memory request / limit | +| --- | --- | --- | --- | +| Coordinator | 1 | 4 / 8 | 16Gi / 32Gi | +| Ingress + API Server | 4, expandable to 8 | 4 / 8 each | 8Gi / 16Gi each | +| Redis, persistent single instance | 1, no replicas or Sentinel | 4 / 8 | 32Gi / 64Gi | + +The four-API profile requests 24 CPU and 80Gi total for these main containers; +eight APIs request 40 CPU and 112Gi. These sums exclude sidecars and node +reservations. Limits allow bursts only where physical capacity remains available. + +API Pods have **required hostname anti-affinity**: at least four eligible nodes +are necessary for four replicas, and eight for eight. Zone spread is preferred, +not required. Provision node capacity for eight API replicas and a node failure +before accepting the corresponding traffic. Scaling the replica count does not +provision that capacity, and readiness probes do not qualify connection draining +or client reconnection. The chart updates multi-replica APIs with zero surge and +one unavailable replica; its PDB retains `replicas - 1` during voluntary eviction. + +Use site values to select a stable control pool for Coordinator, APIs and Redis; +keep them off autoscaled execution-only nodes. Each role accepts `nodeSelector`, +`tolerations` and `affinity`. Account for Redis PV zone/node constraints, and place +Coordinator and Redis on separate hosts where feasible. A single Coordinator +must remain single-active and uses Recreate; the chart rejects multiple replicas. + +## SSD and Redis persistence + +The profile requests a **200Gi ReadWriteOnce PVC** and AOF with +`appendfsync=everysec`. It intentionally leaves `storageClassName` empty because +there is no portable SSD StorageClass name. **Set it explicitly to a qualified +SSD-backed cloud-disk class before installation**; the cluster default is not +proof of SSD performance. Verify topology, binding mode, expansion support, +latency/IOPS and reclaim policy for that class. + +Single-instance Redis means a Redis/host/zone outage can interrupt the entire +control plane until recovery; this is not HA. `everysec` is not zero-RPO, and AOF +on one volume is not an independent backup. Maintain and test backups separately. +Four requested CPU cores reserve scheduling capacity but do not make one Redis +command execute on four cores. Monitor process RSS, dataset size, memory peaks +and fork copy-on-write during persistence/rewrite; the 32Gi request is not a +Redis dataset cap, and a 64Gi Pod limit is not proof that every rewrite fits. +Preserve metadata rather than introducing an eviction policy to hide pressure. + +## Install and render + +Prepare protected site values with the actual image/digest, existing TLS/bootstrap +Secret contract, eligible control-pool placement and SSD class. No image, +credential or provider-specific disk class is supplied by this profile. +From the repository root, review the core chart first: + +```bash +helm template akernel deploy/akernel/charts/core -n akernel \ + -f deploy/akernel/charts/core/values-production.yaml \ + -f /secure/site-values.yaml +``` + +For the umbrella chart, values must be nested under `core`. Generate a wrapper +using Python with PyYAML, then merge the site values **last**: + +```bash +python3 - <<'PY' > /secure/production-umbrella.yaml +import yaml +with open('deploy/akernel/charts/core/values-production.yaml') as source: + print(yaml.safe_dump({'core': yaml.safe_load(source)}, sort_keys=False)) +PY +helm template akernel deploy/akernel -n akernel \ + -f /secure/production-umbrella.yaml -f /secure/site-umbrella-values.yaml +``` + +Both cloud Terraform modules also accept `core_values_override_files`, whose files +are **unwrapped core values**. Append the profile and site overrides after the +module-generated values, in that order. Preserve any existing override files; +review the full Terraform plan because guided node sizing is independent of +these Pod requests. Never feed an umbrella `core:` wrapper into this input. + +To expand APIs, override `adx.ingressApi.replicas: 8` (or +`core.adx.ingressApi.replicas` in umbrella values) after qualifying the additional +hosts, Redis/Coordinator capacity and connection behavior. This profile does not +install an HPA or change the worker autoscaler's policies. + +## Existing-release migration and rollback + +This is not a drop-in live upgrade. Save the current release/chart, complete +rendering chain, values, resource identities and tested rollback configuration in +a protected location. Review live credentials and storage before making changes; +never replace them with defaults. Offline `helm template` cannot recover a Redis +password that the chart normally preserves through live Secret lookup. + +For an existing Redis StatefulSet, changing `volumeClaimTemplates` size from its +current value to 200Gi is immutable, even if the StorageClass supports expansion. +Do not apply this profile blindly. Use an explicitly reviewed storage migration: +retain the existing claim-template value in site overrides, expand the **actual +bound PVC** through the supported CSI procedure, and verify filesystem expansion +and usable capacity. Keep the template override for subsequent upgrades. If the +class cannot expand or must change, plan an offline backup/restore to new storage +with data validation and retention of the original volume; do not delete the old +PVC to make Helm succeed. Never request a shrink for an existing larger claim. + +Drain affected consumers before single-active Coordinator or Redis maintenance, +pause worker scaling actions as appropriate, and preserve scaler state. Review +server-side Helm dry-run and live differences before applying during a maintenance +window. Verify Redis authentication/persistence, API readiness on each host and +actual SDK traffic before resuming scaling and traffic. Existing sessions may +reconnect or be interrupted; test that behavior for the deployed package. + +Rollback resource/replica/placement fields using fresh values while preserving +credentials, data and the current PVC size. A storage expansion is not undone by +Helm rollback. Do not start two Coordinators or replay an old release that changes +credentials or immutable storage fields. + +## Qualification before claiming target capacity + +Measure increasing sandbox inventory separately from connection count and churn. +Exercise both SDKs, command/file/PTY/port streams, reconnects, create/delete bursts, +worker scale-out/in and checkpoint-driven lifecycle load. Include paused instances, +snapshots and metadata retention in Redis sizing. Hold representative steady +traffic and persistence/rewrite cycles; inject API/worker failure and measure the +accepted single-instance Redis/Coordinator interruption and recovery. + +Observe Redis command latency/RSS, persistence/fork peaks, disk growth and I/O; +Coordinator reconcile/scheduling latency; per-API active connections, CPU limiting, +memory and errors; host file-descriptor/connection-tracking budgets, LB limits and +network throughput. Set load-specific acceptance limits and verify graceful +connection drain. Rendering tests establish configuration consistency only. From 9d32699ad139e7efddbd3fecc4a59717082facca Mon Sep 17 00:00:00 2001 From: wayne Date: Mon, 9 Nov 2026 21:47:37 +0800 Subject: [PATCH 2/2] test(deploy): consolidate production profile contracts into chart CI Move the profile rendering, host separation and Redis placement coverage into the chart contract suite already executed by CI. Remove the standalone test file and duplicated Redis/default-value assertions. Keep the capacity guide focused on reusable deployment and storage migration, clarify preservation of all immutable claim-template fields, and leave sizing values in the profile without implying an unmeasured scale guarantee. Signed-off-by: wayne --- AGENTS.md | 6 +- .../charts/core/tests/test_adx_chart.py | 65 +++++- .../core/tests/test_production_capacity.py | 96 -------- deploy/docs/control-plane-capacity.md | 210 ++++++++---------- 4 files changed, 154 insertions(+), 223 deletions(-) delete mode 100644 deploy/akernel/charts/core/tests/test_production_capacity.py diff --git a/AGENTS.md b/AGENTS.md index 3734581f..1a0ad9aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,12 +288,8 @@ environment overrides take precedence. Preserve this handoff when changing node deployment: otherwise the service can fall back to standalone configuration even though the Pod declares a node configuration. -Large deployments may opt into the core chart's `values-production.yaml`. The [control-plane capacity guide](deploy/docs/control-plane-capacity.md) owns -resource budgets, strict API host separation, SSD selection and existing-PVC -migration. These are starting allocations, not a 100,000-sandbox capacity claim. -Keep Redis placement optional and do not apply a capacity profile to a live -release without reviewing current values, data and rollout requirements. +production profile sizing, placement and existing-PVC migration requirements. The managed Kubernetes Redis uses its own Helm-generated authentication Secret, preserved by live lookup during upgrades. Its NetworkPolicy is additional diff --git a/deploy/akernel/charts/core/tests/test_adx_chart.py b/deploy/akernel/charts/core/tests/test_adx_chart.py index 9f59dd39..3c19540f 100755 --- a/deploy/akernel/charts/core/tests/test_adx_chart.py +++ b/deploy/akernel/charts/core/tests/test_adx_chart.py @@ -4,6 +4,7 @@ from __future__ import annotations +import copy import os import shutil import subprocess @@ -16,19 +17,21 @@ CHART = Path(__file__).resolve().parents[1] -def render(*settings: str) -> tuple[list[dict], str]: +def render(*settings: str, values: dict | None = None, umbrella: bool = False) -> tuple[list[dict], str]: result = subprocess.run( [ "helm", "template", "akernel", - str(CHART), + str(CHART.parents[1] if umbrella else CHART), "--namespace", "akernel-system", "--set", - "adx.tls.existingSecret=adx-test-tls", + ("core." if umbrella else "") + "adx.tls.existingSecret=adx-test-tls", + "-f", "-", *[argument for value in settings for argument in ("--set", value)], ], + input=yaml.safe_dump(values or {}), check=True, capture_output=True, text=True, @@ -692,6 +695,62 @@ def test_adx_mode_does_not_render_unused_legacy_control_secrets(self) -> None: self.assertNotIn("akernel-component-tls", names) self.assertNotIn("akernel-master-secret", names) + def test_production_profile_renders_in_core_and_umbrella(self) -> None: + profile = yaml.safe_load((CHART / "values-production.yaml").read_text()) + for umbrella in (False, True): + with self.subTest(umbrella=umbrella): + values = {"core": profile, "monitor": {"enabled": False}} if umbrella else profile + resources, _ = render(values=values, umbrella=umbrella) + objects = {(r["kind"], r["metadata"]["name"]): r for r in resources} + for kind, name, role in ( + ("Deployment", "akernel-adx-coordinator", "coordinator"), + ("Deployment", "akernel-adx-ingress-api", "ingressApi"), + ("StatefulSet", "akernel-adx-redis", "redis"), + ): + spec = objects[kind, name]["spec"] + self.assertEqual(spec["template"]["spec"]["containers"][0]["resources"], + profile["adx"][role]["resources"]) + self.assertEqual(objects["StatefulSet", "akernel-adx-redis"]["spec"] + ["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], + profile["adx"]["redis"]["persistence"]["size"]) + for kind, name in (("DaemonSet", "akernel-node"), ("ConfigMap", "akernel-adx-config")): + self.assertEqual(objects[kind, name], self.resource(kind, name)) + + def test_production_api_host_separation_and_disruption_budget(self) -> None: + for replicas in (4, 8): + with self.subTest(replicas=replicas): + resources, _ = render(f"adx.ingressApi.replicas={replicas}", + values=yaml.safe_load((CHART / "values-production.yaml").read_text())) + objects = {(r["kind"], r["metadata"]["name"]): r for r in resources} + api = objects["Deployment", "akernel-adx-ingress-api"]["spec"] + self.assertEqual(api["replicas"], replicas) + pod = api["template"]["spec"] + rule = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0] + self.assertEqual(rule["topologyKey"], "kubernetes.io/hostname") + self.assertEqual(rule["labelSelector"]["matchLabels"], api["selector"]["matchLabels"]) + self.assertEqual(pod["topologySpreadConstraints"][0]["whenUnsatisfiable"], "ScheduleAnyway") + self.assertEqual(api["strategy"]["rollingUpdate"], {"maxSurge": 0, "maxUnavailable": 1}) + self.assertEqual(objects["PodDisruptionBudget", "akernel-adx-ingress-api"]["spec"]["minAvailable"], + replicas - 1) + + def test_redis_placement_preserves_storage_and_container_configuration(self) -> None: + placement = { + "nodeSelector": {"pool": "control"}, + "tolerations": [{"key": "control", "operator": "Exists", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [{ + "weight": 100, "podAffinityTerm": { + "topologyKey": "kubernetes.io/hostname", + "labelSelector": {"matchLabels": {"app": "akernel-adx-coordinator"}}, + }, + }]}}, + } + resources, _ = render(values={"adx": {"redis": placement}}) + actual = next(r for r in resources if r["kind"] == "StatefulSet" and + r["metadata"]["name"] == "akernel-adx-redis") + expected = copy.deepcopy(self.resource("StatefulSet", "akernel-adx-redis")) + expected["spec"]["template"]["spec"].update(placement) + self.assertEqual(actual, expected) + def test_multiple_coordinator_replicas_are_rejected(self) -> None: result = subprocess.run( [ diff --git a/deploy/akernel/charts/core/tests/test_production_capacity.py b/deploy/akernel/charts/core/tests/test_production_capacity.py deleted file mode 100644 index bb04937f..00000000 --- a/deploy/akernel/charts/core/tests/test_production_capacity.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Production profile placement and compatibility contracts; no scale claim.""" -import copy -import subprocess -import unittest -from pathlib import Path - -import yaml - -CHART = Path(__file__).resolve().parents[1] -PROFILE = yaml.safe_load((CHART / "values-production.yaml").read_text()) - - -def render(values, umbrella=False): - chart = CHART.parents[1] if umbrella else CHART - result = subprocess.run( - ["helm", "template", "akernel", str(chart), "-n", "akernel", "-f", "-"], - input=yaml.safe_dump(values), text=True, capture_output=True, check=True, - ) - return {(x["kind"], x["metadata"]["name"]): x - for x in yaml.safe_load_all(result.stdout) if x} - - -class ProductionCapacityTests(unittest.TestCase): - def test_profile_keeps_single_active_coordinator_and_persistent_redis(self): - objects = render(PROFILE) - coordinator = objects["Deployment", "akernel-adx-coordinator"]["spec"] - self.assertEqual(coordinator["replicas"], 1) - self.assertEqual(coordinator["strategy"]["type"], "Recreate") - redis = objects["StatefulSet", "akernel-adx-redis"]["spec"] - self.assertEqual(redis["replicas"], 1) - container = redis["template"]["spec"]["containers"][0] - self.assertEqual(container["args"][container["args"].index("--appendonly") + 1], "yes") - self.assertEqual(container["args"][container["args"].index("--appendfsync") + 1], "everysec") - self.assertEqual(container["env"][0]["valueFrom"]["secretKeyRef"]["name"], "akernel-adx-redis-auth") - claim = redis["volumeClaimTemplates"][0]["spec"] - self.assertEqual(claim["resources"]["requests"]["storage"], "200Gi") - self.assertEqual(claim["accessModes"], ["ReadWriteOnce"]) - self.assertNotIn("storageClassName", claim) # operator must select qualified SSD - - def test_api_host_separation_scales_with_replica_count_and_pdb(self): - for replicas in (4, 8): - values = copy.deepcopy(PROFILE) - values["adx"]["ingressApi"]["replicas"] = replicas - objects = render(values) - api = objects["Deployment", "akernel-adx-ingress-api"]["spec"] - self.assertEqual(api["replicas"], replicas) - pod = api["template"]["spec"] - rule = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0] - self.assertEqual(rule["topologyKey"], "kubernetes.io/hostname") - self.assertEqual(rule["labelSelector"]["matchLabels"], api["selector"]["matchLabels"]) - self.assertEqual(pod["topologySpreadConstraints"][0]["whenUnsatisfiable"], "ScheduleAnyway") - self.assertEqual(api["strategy"]["rollingUpdate"], {"maxSurge": 0, "maxUnavailable": 1}) - self.assertEqual(objects["PodDisruptionBudget", "akernel-adx-ingress-api"]["spec"]["minAvailable"], replicas - 1) - - def test_profile_changes_only_capacity_and_api_placement(self): - before, after = render({}), render(PROFILE) - allowed = {("Deployment", "akernel-adx-coordinator"), - ("Deployment", "akernel-adx-ingress-api"), - ("StatefulSet", "akernel-adx-redis"), - ("PodDisruptionBudget", "akernel-adx-ingress-api"), - ("Secret", "akernel-adx-redis-auth")} # offline random password - self.assertEqual(after.keys() - before.keys(), {("PodDisruptionBudget", "akernel-adx-ingress-api")}) - for key in before.keys() - allowed: - self.assertEqual(before[key], after[key], key) - for kind, name, role in (("Deployment", "akernel-adx-coordinator", "coordinator"), - ("Deployment", "akernel-adx-ingress-api", "ingressApi"), - ("StatefulSet", "akernel-adx-redis", "redis")): - pod = after[kind, name]["spec"]["template"]["spec"] - self.assertEqual(pod["containers"][0]["resources"], PROFILE["adx"][role]["resources"]) - - def test_redis_placement_and_storage_site_overrides(self): - values = copy.deepcopy(PROFILE) - placement = {"nodeSelector": {"pool": "control"}, - "tolerations": [{"key": "control", "operator": "Exists", "effect": "NoSchedule"}], - "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": []}}} - values["adx"]["redis"].update(placement) - values["adx"]["redis"]["persistence"].update(storageClassName="qualified-ssd", size="300Gi") - spec = render(values)["StatefulSet", "akernel-adx-redis"]["spec"] - for key, value in placement.items(): - self.assertEqual(spec["template"]["spec"][key], value) - self.assertEqual(spec["volumeClaimTemplates"][0]["spec"]["storageClassName"], "qualified-ssd") - self.assertEqual(spec["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], "300Gi") - baseline = render({})["StatefulSet", "akernel-adx-redis"]["spec"]["template"]["spec"] - for key in placement: - self.assertNotIn(key, baseline) - - def test_umbrella_uses_explicit_core_wrapper_and_retains_light_defaults(self): - objects = render({"core": PROFILE, "monitor": {"enabled": False}}, umbrella=True) - self.assertEqual(objects["Deployment", "akernel-adx-ingress-api"]["spec"]["replicas"], 4) - default = render({}) - self.assertEqual(default["Deployment", "akernel-adx-ingress-api"]["spec"]["replicas"], 1) - self.assertEqual(default["StatefulSet", "akernel-adx-redis"]["spec"]["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], "20Gi") - - -if __name__ == "__main__": - unittest.main() diff --git a/deploy/docs/control-plane-capacity.md b/deploy/docs/control-plane-capacity.md index 04de9eaf..f6708c4f 100644 --- a/deploy/docs/control-plane-capacity.md +++ b/deploy/docs/control-plane-capacity.md @@ -1,140 +1,112 @@ -# Production control-plane capacity profile +# Control-plane capacity and placement -The core chart's [values-production.yaml](../akernel/charts/core/values-production.yaml) -is an opt-in starting allocation for installations targeting roughly 100,000 -running sandboxes and 10,000 active connections. These targets are **not measured -capacity guarantees**. The default values remain suitable for smaller installations; -this profile does not change standalone deployment or provision cloud nodes. +Use the core chart's [values-production.yaml](../akernel/charts/core/values-production.yaml) +as an **opt-in sizing example** for larger installations. The values file owns +the resource allocations; this guide owns deployment, placement and migration. +The profile is not a measured sandbox-count or connection-count capacity guarantee. +It leaves the lightweight defaults, runtime configuration and worker pool unchanged. -Ingress and API Server run together in each API Pod. Both protocols continue to -share one control plane and worker pool. This profile neither enables E2B nor -changes TLS, credentials, public Services, worker resources or checkpoint storage. +## Resource budget and placement -## Allocation and placement - -Each row describes the main container; budget separately for collectors, system -services, monitoring, image distribution and workload nodes. - -| Component | Replicas | CPU request / limit | Memory request / limit | +| Main container | Replicas | CPU request / limit | Memory request / limit | | --- | --- | --- | --- | | Coordinator | 1 | 4 / 8 | 16Gi / 32Gi | | Ingress + API Server | 4, expandable to 8 | 4 / 8 each | 8Gi / 16Gi each | -| Redis, persistent single instance | 1, no replicas or Sentinel | 4 / 8 | 32Gi / 64Gi | - -The four-API profile requests 24 CPU and 80Gi total for these main containers; -eight APIs request 40 CPU and 112Gi. These sums exclude sidecars and node -reservations. Limits allow bursts only where physical capacity remains available. - -API Pods have **required hostname anti-affinity**: at least four eligible nodes -are necessary for four replicas, and eight for eight. Zone spread is preferred, -not required. Provision node capacity for eight API replicas and a node failure -before accepting the corresponding traffic. Scaling the replica count does not -provision that capacity, and readiness probes do not qualify connection draining -or client reconnection. The chart updates multi-replica APIs with zero surge and -one unavailable replica; its PDB retains `replicas - 1` during voluntary eviction. - -Use site values to select a stable control pool for Coordinator, APIs and Redis; -keep them off autoscaled execution-only nodes. Each role accepts `nodeSelector`, -`tolerations` and `affinity`. Account for Redis PV zone/node constraints, and place -Coordinator and Redis on separate hosts where feasible. A single Coordinator -must remain single-active and uses Recreate; the chart rejects multiple replicas. - -## SSD and Redis persistence - -The profile requests a **200Gi ReadWriteOnce PVC** and AOF with -`appendfsync=everysec`. It intentionally leaves `storageClassName` empty because -there is no portable SSD StorageClass name. **Set it explicitly to a qualified -SSD-backed cloud-disk class before installation**; the cluster default is not -proof of SSD performance. Verify topology, binding mode, expansion support, -latency/IOPS and reclaim policy for that class. - -Single-instance Redis means a Redis/host/zone outage can interrupt the entire -control plane until recovery; this is not HA. `everysec` is not zero-RPO, and AOF -on one volume is not an independent backup. Maintain and test backups separately. -Four requested CPU cores reserve scheduling capacity but do not make one Redis -command execute on four cores. Monitor process RSS, dataset size, memory peaks -and fork copy-on-write during persistence/rewrite; the 32Gi request is not a -Redis dataset cap, and a 64Gi Pod limit is not proof that every rewrite fits. -Preserve metadata rather than introducing an eviction policy to hide pressure. - -## Install and render - -Prepare protected site values with the actual image/digest, existing TLS/bootstrap -Secret contract, eligible control-pool placement and SSD class. No image, -credential or provider-specific disk class is supplied by this profile. -From the repository root, review the core chart first: +| Managed Redis | 1 | 4 / 8 | 32Gi / 64Gi | + +These containers request 24 CPU / 80Gi with four API replicas, or 40 CPU / 112Gi +with eight. Budget separately for collectors, system reservations, monitoring, +image distribution and workers. Limits permit bursts only where physical +capacity remains available; requests are scheduling reservations, not usage caps. + +API replicas require distinct hosts: four eligible nodes for four replicas, +eight for eight. Provision additional headroom to retain that replica count +through a node failure. Zone spread is preferred. The existing rolling update +uses zero surge / one unavailable replica, and the PDB permits one voluntary +disruption; neither setting proves that active connections drain gracefully. + +Select a stable control pool through each role's `nodeSelector`, `tolerations` +and `affinity`, rather than placing control services on elastic execution-only +nodes. Redis placement must also satisfy its PV topology. Keep Coordinator and +Redis on separate hosts where feasible. Coordinator remains single-active with +Recreate; additional API replicas do not remove Coordinator or Redis outages. + +## Applying the profile + +Prepare site values for the actual image/digest, TLS/bootstrap Secret, control +pool and an explicitly selected SSD-backed StorageClass. Use the existing +[Kubernetes installation contract](../README.md#2-kubernetes-helm) for credentials, +images and public Services. The profile does not enable E2B or alter its ingress +policy; protocol enablement remains a separate deployment choice. + +Render the core chart from the repository root, merging site values last: ```bash helm template akernel deploy/akernel/charts/core -n akernel \ -f deploy/akernel/charts/core/values-production.yaml \ - -f /secure/site-values.yaml + -f /path/to/site-values.yaml ``` -For the umbrella chart, values must be nested under `core`. Generate a wrapper -using Python with PyYAML, then merge the site values **last**: +The umbrella chart expects these values under `core`. Generate a wrapper with +Python/PyYAML, then render with your complete umbrella site values: ```bash -python3 - <<'PY' > /secure/production-umbrella.yaml +python3 - <<'PY' > /path/to/production-umbrella.yaml import yaml with open('deploy/akernel/charts/core/values-production.yaml') as source: print(yaml.safe_dump({'core': yaml.safe_load(source)}, sort_keys=False)) PY helm template akernel deploy/akernel -n akernel \ - -f /secure/production-umbrella.yaml -f /secure/site-umbrella-values.yaml + -f /path/to/production-umbrella.yaml -f /path/to/site-umbrella-values.yaml ``` -Both cloud Terraform modules also accept `core_values_override_files`, whose files -are **unwrapped core values**. Append the profile and site overrides after the -module-generated values, in that order. Preserve any existing override files; -review the full Terraform plan because guided node sizing is independent of -these Pod requests. Never feed an umbrella `core:` wrapper into this input. - -To expand APIs, override `adx.ingressApi.replicas: 8` (or -`core.adx.ingressApi.replicas` in umbrella values) after qualifying the additional -hosts, Redis/Coordinator capacity and connection behavior. This profile does not -install an HPA or change the worker autoscaler's policies. - -## Existing-release migration and rollback - -This is not a drop-in live upgrade. Save the current release/chart, complete -rendering chain, values, resource identities and tested rollback configuration in -a protected location. Review live credentials and storage before making changes; -never replace them with defaults. Offline `helm template` cannot recover a Redis -password that the chart normally preserves through live Secret lookup. - -For an existing Redis StatefulSet, changing `volumeClaimTemplates` size from its -current value to 200Gi is immutable, even if the StorageClass supports expansion. -Do not apply this profile blindly. Use an explicitly reviewed storage migration: -retain the existing claim-template value in site overrides, expand the **actual -bound PVC** through the supported CSI procedure, and verify filesystem expansion -and usable capacity. Keep the template override for subsequent upgrades. If the -class cannot expand or must change, plan an offline backup/restore to new storage -with data validation and retention of the original volume; do not delete the old -PVC to make Helm succeed. Never request a shrink for an existing larger claim. - -Drain affected consumers before single-active Coordinator or Redis maintenance, -pause worker scaling actions as appropriate, and preserve scaler state. Review -server-side Helm dry-run and live differences before applying during a maintenance -window. Verify Redis authentication/persistence, API readiness on each host and -actual SDK traffic before resuming scaling and traffic. Existing sessions may -reconnect or be interrupted; test that behavior for the deployed package. - -Rollback resource/replica/placement fields using fresh values while preserving -credentials, data and the current PVC size. A storage expansion is not undone by -Helm rollback. Do not start two Coordinators or replay an old release that changes -credentials or immutable storage fields. - -## Qualification before claiming target capacity - -Measure increasing sandbox inventory separately from connection count and churn. -Exercise both SDKs, command/file/PTY/port streams, reconnects, create/delete bursts, -worker scale-out/in and checkpoint-driven lifecycle load. Include paused instances, -snapshots and metadata retention in Redis sizing. Hold representative steady -traffic and persistence/rewrite cycles; inject API/worker failure and measure the -accepted single-instance Redis/Coordinator interruption and recovery. - -Observe Redis command latency/RSS, persistence/fork peaks, disk growth and I/O; -Coordinator reconcile/scheduling latency; per-API active connections, CPU limiting, -memory and errors; host file-descriptor/connection-tracking budgets, LB limits and -network throughput. Set load-specific acceptance limits and verify graceful -connection drain. Rendering tests establish configuration consistency only. +Both cloud Terraform modules accept `core_values_override_files`: append the +**unwrapped** profile and site values after any existing overrides, in that order. +These files override module-generated Pod values, not cloud-node sizing; inspect +the full plan. To expand APIs, set `adx.ingressApi.replicas: 8` (under `core` for +the umbrella chart) after qualifying the extra hosts and backend capacity. +The profile installs no HPA and makes no changes to worker scaling policies. + +## Redis storage and existing releases + +The profile selects one Redis instance, AOF `appendfsync=everysec`, and a 200Gi +ReadWriteOnce PVC. Set `adx.redis.persistence.storageClassName` explicitly to a +qualified SSD-backed cloud-disk class. The empty value uses the cluster default, +which does not establish SSD performance. Verify the class's topology, binding, +expansion and reclaim behavior, and qualify latency/IOPS for persistence. + +An existing StatefulSet's `volumeClaimTemplates` cannot simply change to 200Gi +through Helm. Preserve **all existing immutable claim-template fields** in site +overrides, including size, StorageClass and access modes. Expand the actual bound +PVC through a supported CSI procedure and verify filesystem capacity; retain the +template overrides on later upgrades. Do not shrink an already larger claim. +If expansion or a required class change needs new storage, plan an offline +backup/restore with data verification and retain the original volume. Do not +delete a PVC to make a Helm upgrade succeed. + +Before modifying a live release, save its complete chart/values/rendering chain +and verify current Secret and PVC identities. Offline rendering cannot recover +the Redis password preserved by live Helm lookup. Review server-side dry-run +and live differences, drain affected consumers for Coordinator/Redis maintenance, +and pause/resume scaler actions as needed without clearing their state. Verify +Redis authentication/persistence, each API Pod and real SDK traffic before +restoring traffic. Roll back only reviewed resource/replica/placement fields; +preserve credentials, data and the actual PVC size. Helm rollback cannot undo +volume expansion, and must never start a second active Coordinator. + +## Capacity qualification + +Single-instance Redis is an accepted availability boundary, not HA. `everysec` +is not zero-RPO, and AOF on one volume is not an independent backup. Maintain +and test backups separately. Redis CPU reservations do not parallelize individual +commands. Monitor RSS, dataset growth and fork copy-on-write during persistence; +the 32Gi request is not a dataset cap and 64Gi does not guarantee every rewrite +fits. Do not introduce metadata eviction to hide memory pressure. + +Qualify inventory, active connections and create/delete churn independently, +including retained snapshots, paused sandboxes and persistence/rewrite cycles. +Exercise both SDKs, streams, reconnect/drain behavior, worker scaling and the +accepted Coordinator/Redis interruption and recovery. Measure scheduling and +API latency/errors, CPU throttling, Redis memory/command latency, disk growth/I/O, +and host/LB connection and file-descriptor limits. Set workload-specific acceptance +limits before claiming supported capacity; chart rendering is not a load test.