diff --git a/AGENTS.md b/AGENTS.md index decf6322..1a0ad9aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -288,6 +288,9 @@ environment overrides take precedence. Preserve this handoff when changing node deployment: otherwise the service can fall back to standalone configuration even though the Pod declares a node configuration. +The [control-plane capacity guide](deploy/docs/control-plane-capacity.md) owns +production profile sizing, placement and existing-PVC migration requirements. + The managed Kubernetes Redis uses its own Helm-generated authentication Secret, preserved by live lookup during upgrades. Its NetworkPolicy is additional isolation and requires CNI enforcement. Coordinator and Ingress/API Server run in separate diff --git a/deploy/README.md b/deploy/README.md index c5c1eb3f..7d640bd2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -476,6 +476,9 @@ The default chart runs ADX Ingress and API Server in one `akernel-adx-ingress-api` Pod, with 2 CPU requested and an 8 CPU limit. Override `core.adx.ingressApi.resources` for your node shape and workload. Measure capacity with the [Create benchmark](../sdk/python/benchmarks/create_throughput/README.md). +For larger installations, use the opt-in [production control-plane capacity +profile](docs/control-plane-capacity.md), including placement, SSD selection, +capacity qualification and existing-PVC migration requirements. The `monitor` subchart remains optional and contains Prometheus, Grafana, Loki, and Tempo. Kubernetes nodes must support privileged Pods and the runtime diff --git a/deploy/akernel/charts/core/templates/adx/redis.yaml b/deploy/akernel/charts/core/templates/adx/redis.yaml index 5fb81773..a44aaf6e 100644 --- a/deploy/akernel/charts/core/templates/adx/redis.yaml +++ b/deploy/akernel/charts/core/templates/adx/redis.yaml @@ -42,6 +42,15 @@ spec: labels: app: akernel-adx-redis spec: + {{- with .Values.adx.redis.nodeSelector }} + nodeSelector: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.adx.redis.tolerations }} + tolerations: {{ toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.adx.redis.affinity }} + affinity: {{ toYaml . | nindent 8 }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/deploy/akernel/charts/core/tests/test_adx_chart.py b/deploy/akernel/charts/core/tests/test_adx_chart.py index 9f59dd39..3c19540f 100755 --- a/deploy/akernel/charts/core/tests/test_adx_chart.py +++ b/deploy/akernel/charts/core/tests/test_adx_chart.py @@ -4,6 +4,7 @@ from __future__ import annotations +import copy import os import shutil import subprocess @@ -16,19 +17,21 @@ CHART = Path(__file__).resolve().parents[1] -def render(*settings: str) -> tuple[list[dict], str]: +def render(*settings: str, values: dict | None = None, umbrella: bool = False) -> tuple[list[dict], str]: result = subprocess.run( [ "helm", "template", "akernel", - str(CHART), + str(CHART.parents[1] if umbrella else CHART), "--namespace", "akernel-system", "--set", - "adx.tls.existingSecret=adx-test-tls", + ("core." if umbrella else "") + "adx.tls.existingSecret=adx-test-tls", + "-f", "-", *[argument for value in settings for argument in ("--set", value)], ], + input=yaml.safe_dump(values or {}), check=True, capture_output=True, text=True, @@ -692,6 +695,62 @@ def test_adx_mode_does_not_render_unused_legacy_control_secrets(self) -> None: self.assertNotIn("akernel-component-tls", names) self.assertNotIn("akernel-master-secret", names) + def test_production_profile_renders_in_core_and_umbrella(self) -> None: + profile = yaml.safe_load((CHART / "values-production.yaml").read_text()) + for umbrella in (False, True): + with self.subTest(umbrella=umbrella): + values = {"core": profile, "monitor": {"enabled": False}} if umbrella else profile + resources, _ = render(values=values, umbrella=umbrella) + objects = {(r["kind"], r["metadata"]["name"]): r for r in resources} + for kind, name, role in ( + ("Deployment", "akernel-adx-coordinator", "coordinator"), + ("Deployment", "akernel-adx-ingress-api", "ingressApi"), + ("StatefulSet", "akernel-adx-redis", "redis"), + ): + spec = objects[kind, name]["spec"] + self.assertEqual(spec["template"]["spec"]["containers"][0]["resources"], + profile["adx"][role]["resources"]) + self.assertEqual(objects["StatefulSet", "akernel-adx-redis"]["spec"] + ["volumeClaimTemplates"][0]["spec"]["resources"]["requests"]["storage"], + profile["adx"]["redis"]["persistence"]["size"]) + for kind, name in (("DaemonSet", "akernel-node"), ("ConfigMap", "akernel-adx-config")): + self.assertEqual(objects[kind, name], self.resource(kind, name)) + + def test_production_api_host_separation_and_disruption_budget(self) -> None: + for replicas in (4, 8): + with self.subTest(replicas=replicas): + resources, _ = render(f"adx.ingressApi.replicas={replicas}", + values=yaml.safe_load((CHART / "values-production.yaml").read_text())) + objects = {(r["kind"], r["metadata"]["name"]): r for r in resources} + api = objects["Deployment", "akernel-adx-ingress-api"]["spec"] + self.assertEqual(api["replicas"], replicas) + pod = api["template"]["spec"] + rule = pod["affinity"]["podAntiAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][0] + self.assertEqual(rule["topologyKey"], "kubernetes.io/hostname") + self.assertEqual(rule["labelSelector"]["matchLabels"], api["selector"]["matchLabels"]) + self.assertEqual(pod["topologySpreadConstraints"][0]["whenUnsatisfiable"], "ScheduleAnyway") + self.assertEqual(api["strategy"]["rollingUpdate"], {"maxSurge": 0, "maxUnavailable": 1}) + self.assertEqual(objects["PodDisruptionBudget", "akernel-adx-ingress-api"]["spec"]["minAvailable"], + replicas - 1) + + def test_redis_placement_preserves_storage_and_container_configuration(self) -> None: + placement = { + "nodeSelector": {"pool": "control"}, + "tolerations": [{"key": "control", "operator": "Exists", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [{ + "weight": 100, "podAffinityTerm": { + "topologyKey": "kubernetes.io/hostname", + "labelSelector": {"matchLabels": {"app": "akernel-adx-coordinator"}}, + }, + }]}}, + } + resources, _ = render(values={"adx": {"redis": placement}}) + actual = next(r for r in resources if r["kind"] == "StatefulSet" and + r["metadata"]["name"] == "akernel-adx-redis") + expected = copy.deepcopy(self.resource("StatefulSet", "akernel-adx-redis")) + expected["spec"]["template"]["spec"].update(placement) + self.assertEqual(actual, expected) + def test_multiple_coordinator_replicas_are_rejected(self) -> None: result = subprocess.run( [ diff --git a/deploy/akernel/charts/core/values-production.yaml b/deploy/akernel/charts/core/values-production.yaml new file mode 100644 index 00000000..582299de --- /dev/null +++ b/deploy/akernel/charts/core/values-production.yaml @@ -0,0 +1,52 @@ +# Opt-in core-chart capacity starting point, NOT a measured capacity guarantee. +# Read deploy/docs/control-plane-capacity.md before installing or upgrading. +# Supply a qualified SSD StorageClass and placement overrides for your cluster. +adx: + coordinator: + replicas: 1 + resources: + requests: + cpu: "4" + memory: 16Gi + limits: + cpu: "8" + memory: 32Gi + ingressApi: + replicas: 4 + resources: + requests: + cpu: "4" + memory: 8Gi + limits: + cpu: "8" + memory: 16Gi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: akernel-adx-ingress-api + topologyKey: kubernetes.io/hostname + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: akernel-adx-ingress-api + redis: + mode: managed + appendfsync: everysec + persistence: + enabled: true + # Must override with a qualified SSD-backed StorageClass; no portable name. + storageClassName: "" + size: 200Gi + accessModes: [ReadWriteOnce] + resources: + requests: + cpu: "4" + memory: 32Gi + limits: + cpu: "8" + memory: 64Gi diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index eab9c69d..aad434dd 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -76,7 +76,11 @@ adx: tls: existingSecret: akernel-adx-tls redis: + # Optional production capacity baseline: values-production.yaml. mode: managed + nodeSelector: {} + tolerations: [] + affinity: {} appendfsync: everysec external: existingSecret: "" diff --git a/deploy/docs/control-plane-capacity.md b/deploy/docs/control-plane-capacity.md new file mode 100644 index 00000000..f6708c4f --- /dev/null +++ b/deploy/docs/control-plane-capacity.md @@ -0,0 +1,112 @@ +# Control-plane capacity and placement + +Use the core chart's [values-production.yaml](../akernel/charts/core/values-production.yaml) +as an **opt-in sizing example** for larger installations. The values file owns +the resource allocations; this guide owns deployment, placement and migration. +The profile is not a measured sandbox-count or connection-count capacity guarantee. +It leaves the lightweight defaults, runtime configuration and worker pool unchanged. + +## Resource budget and placement + +| Main container | Replicas | CPU request / limit | Memory request / limit | +| --- | --- | --- | --- | +| Coordinator | 1 | 4 / 8 | 16Gi / 32Gi | +| Ingress + API Server | 4, expandable to 8 | 4 / 8 each | 8Gi / 16Gi each | +| Managed Redis | 1 | 4 / 8 | 32Gi / 64Gi | + +These containers request 24 CPU / 80Gi with four API replicas, or 40 CPU / 112Gi +with eight. Budget separately for collectors, system reservations, monitoring, +image distribution and workers. Limits permit bursts only where physical +capacity remains available; requests are scheduling reservations, not usage caps. + +API replicas require distinct hosts: four eligible nodes for four replicas, +eight for eight. Provision additional headroom to retain that replica count +through a node failure. Zone spread is preferred. The existing rolling update +uses zero surge / one unavailable replica, and the PDB permits one voluntary +disruption; neither setting proves that active connections drain gracefully. + +Select a stable control pool through each role's `nodeSelector`, `tolerations` +and `affinity`, rather than placing control services on elastic execution-only +nodes. Redis placement must also satisfy its PV topology. Keep Coordinator and +Redis on separate hosts where feasible. Coordinator remains single-active with +Recreate; additional API replicas do not remove Coordinator or Redis outages. + +## Applying the profile + +Prepare site values for the actual image/digest, TLS/bootstrap Secret, control +pool and an explicitly selected SSD-backed StorageClass. Use the existing +[Kubernetes installation contract](../README.md#2-kubernetes-helm) for credentials, +images and public Services. The profile does not enable E2B or alter its ingress +policy; protocol enablement remains a separate deployment choice. + +Render the core chart from the repository root, merging site values last: + +```bash +helm template akernel deploy/akernel/charts/core -n akernel \ + -f deploy/akernel/charts/core/values-production.yaml \ + -f /path/to/site-values.yaml +``` + +The umbrella chart expects these values under `core`. Generate a wrapper with +Python/PyYAML, then render with your complete umbrella site values: + +```bash +python3 - <<'PY' > /path/to/production-umbrella.yaml +import yaml +with open('deploy/akernel/charts/core/values-production.yaml') as source: + print(yaml.safe_dump({'core': yaml.safe_load(source)}, sort_keys=False)) +PY +helm template akernel deploy/akernel -n akernel \ + -f /path/to/production-umbrella.yaml -f /path/to/site-umbrella-values.yaml +``` + +Both cloud Terraform modules accept `core_values_override_files`: append the +**unwrapped** profile and site values after any existing overrides, in that order. +These files override module-generated Pod values, not cloud-node sizing; inspect +the full plan. To expand APIs, set `adx.ingressApi.replicas: 8` (under `core` for +the umbrella chart) after qualifying the extra hosts and backend capacity. +The profile installs no HPA and makes no changes to worker scaling policies. + +## Redis storage and existing releases + +The profile selects one Redis instance, AOF `appendfsync=everysec`, and a 200Gi +ReadWriteOnce PVC. Set `adx.redis.persistence.storageClassName` explicitly to a +qualified SSD-backed cloud-disk class. The empty value uses the cluster default, +which does not establish SSD performance. Verify the class's topology, binding, +expansion and reclaim behavior, and qualify latency/IOPS for persistence. + +An existing StatefulSet's `volumeClaimTemplates` cannot simply change to 200Gi +through Helm. Preserve **all existing immutable claim-template fields** in site +overrides, including size, StorageClass and access modes. Expand the actual bound +PVC through a supported CSI procedure and verify filesystem capacity; retain the +template overrides on later upgrades. Do not shrink an already larger claim. +If expansion or a required class change needs new storage, plan an offline +backup/restore with data verification and retain the original volume. Do not +delete a PVC to make a Helm upgrade succeed. + +Before modifying a live release, save its complete chart/values/rendering chain +and verify current Secret and PVC identities. Offline rendering cannot recover +the Redis password preserved by live Helm lookup. Review server-side dry-run +and live differences, drain affected consumers for Coordinator/Redis maintenance, +and pause/resume scaler actions as needed without clearing their state. Verify +Redis authentication/persistence, each API Pod and real SDK traffic before +restoring traffic. Roll back only reviewed resource/replica/placement fields; +preserve credentials, data and the actual PVC size. Helm rollback cannot undo +volume expansion, and must never start a second active Coordinator. + +## Capacity qualification + +Single-instance Redis is an accepted availability boundary, not HA. `everysec` +is not zero-RPO, and AOF on one volume is not an independent backup. Maintain +and test backups separately. Redis CPU reservations do not parallelize individual +commands. Monitor RSS, dataset growth and fork copy-on-write during persistence; +the 32Gi request is not a dataset cap and 64Gi does not guarantee every rewrite +fits. Do not introduce metadata eviction to hide memory pressure. + +Qualify inventory, active connections and create/delete churn independently, +including retained snapshots, paused sandboxes and persistence/rewrite cycles. +Exercise both SDKs, streams, reconnect/drain behavior, worker scaling and the +accepted Coordinator/Redis interruption and recovery. Measure scheduling and +API latency/errors, CPU throttling, Redis memory/command latency, disk growth/I/O, +and host/LB connection and file-descriptor limits. Set workload-specific acceptance +limits before claiming supported capacity; chart rendering is not a load test.