diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b793ca8..66215efb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,7 @@ jobs: - name: Test standalone deployment helpers run: | + python3 -m pip install 'tomli>=2,<3' make deploy-standalone-test make build-helper-test adx-integration-contracts: diff --git a/AGENTS.md b/AGENTS.md index b7b468d4..decf6322 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -247,6 +247,32 @@ distill-fs release's packaged manifest. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and Terraform-based cloud provisioning. +The direct Helm chart enables the cgroup v2 parent memory policy with +`total_reserve="16GiB"` for the ordinary KVM profile. Standalone and both cloud +Terraform profiles keep it disabled by default. Cloud opt-in uses +`sandbox_memory_limit_enabled`; guided profiles expose +`SANDBOX_MEMORY_LIMIT_ENABLED` and `SANDBOX_MEMORY_TOTAL_RESERVE`. Qualify cgroup +v2 and effective memory (host MemTotal clipped by visible ancestor limits) +above the reserve with room for sandbox requests before enabling the policy. +The bundled standalone ADX profile retains its `auto` resource source, which +does not guarantee scheduling capacity follows the sandbox parent ceiling. +Operators enabling the policy must select and qualify a compatible +`resource_source`, including memory capacity, allocation/release accounting and +schedulable CPU. The disabled default does not fix resource-source limitations. +PVM also keeps the policy disabled until explicitly selected with +`AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` in standalone or +`node.config.sandboxd.pvmMemoryLimitEnabled=true` in Helm. Size the reserve for +the selected L1 and qualify the finite-cap and scheduling combination on the +matching PVM host before rollout; ordinary runsc tests do not qualify PVM. +Validate PVM memory switches against parsed TOML. PVM chart rendering requires +Helm 3.17 or newer, and standalone uses Python 3.11 or Python 3.10 with +`tomli`. Reject a standalone rewrite unless the parsed configuration matches +the requested policy and preserves the other fields. +Adding or lowering a parent cap requires an empty sandbox subtree; an existing +stricter cap is retained. This aggregate sandbox cap does not bound all node +components or guarantee avoidance of host OOM. +See `deploy/README.md#aggregate-sandbox-memory-budget` for migration and scope. + The [standalone guide](./deploy/standalone/README.md) owns source startup, supported platforms, runtime selection, and SDK setup. Preserve the source launcher's native-daemon architecture selection, clean sandboxd gitlink check, occupied-name preflight, profile locking, unique source image, and node/gateway health checks. Status and stop must verify recorded container identities, images, and data mounts; stop also requires an empty sandbox inventory. Source management is independent of cloud profiles and must preserve unrelated containers. The all-in-one image and node launchers declare lowercase `container=oci` diff --git a/Makefile b/Makefile index 37abffdd..8bd442b0 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,8 @@ INSTALL_MONITOR ?= INSTALL_DRAGONFLY ?= NODE_IMAGE_PULL_MODE ?= CHUNK_DB_SIZE ?= +SANDBOX_MEMORY_LIMIT_ENABLED ?= +SANDBOX_MEMORY_TOTAL_RESERVE ?= JUICEFS_ENABLED ?= false JUICEFS_STORAGE_CLASS ?= JUICEFS_CACHE_MIB ?= 4096 @@ -52,6 +54,7 @@ help: @echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution" @echo " make config NODE_IMAGE_PULL_MODE=direct Use direct registry pulls on new ACK workers" @echo " make config ENABLE_RUNC=true Build and register the optional runc runtime" + @echo " make config SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB Opt into a qualified cgroup v2 sandbox budget" @echo " make config JUICEFS_ENABLED=true JUICEFS_STORAGE_CLASS=... Deploy JuiceFS/PostgreSQL/Ceph" @echo " make config CHECKPOINT_STORAGE_HOST_PATH=/srv/shared Use an existing AFS/JuiceFS mount" @echo " make config SCHEDULE_PLACEMENT_POLICY=binpack Use compact scheduling" @@ -105,6 +108,8 @@ config: if [[ -n "$(INSTALL_DRAGONFLY)" ]]; then args+=(--install-dragonfly "$(INSTALL_DRAGONFLY)"); fi; \ if [[ -n "$(NODE_IMAGE_PULL_MODE)" ]]; then args+=(--node-image-pull-mode "$(NODE_IMAGE_PULL_MODE)"); fi; \ if [[ -n "$(CHUNK_DB_SIZE)" ]]; then args+=(--chunk-db-size "$(CHUNK_DB_SIZE)"); fi; \ + if [[ -n "$(SANDBOX_MEMORY_LIMIT_ENABLED)" ]]; then args+=(--sandbox-memory-limit-enabled "$(SANDBOX_MEMORY_LIMIT_ENABLED)"); fi; \ + if [[ -n "$(SANDBOX_MEMORY_TOTAL_RESERVE)" ]]; then args+=(--sandbox-memory-total-reserve "$(SANDBOX_MEMORY_TOTAL_RESERVE)"); fi; \ args+=(--juicefs-enabled "$(JUICEFS_ENABLED)"); \ args+=(--juicefs-cache-mib "$(JUICEFS_CACHE_MIB)" --juicefs-cache-large-write "$(JUICEFS_CACHE_LARGE_WRITE)"); \ if [[ -n "$(JUICEFS_CACHE_HOST_PATH)" ]]; then args+=(--juicefs-cache-host-path "$(JUICEFS_CACHE_HOST_PATH)"); fi; \ diff --git a/builder/scripts/test_afs_config.py b/builder/scripts/test_afs_config.py index cb32999b..38e6de76 100644 --- a/builder/scripts/test_afs_config.py +++ b/builder/scripts/test_afs_config.py @@ -30,6 +30,36 @@ def assert_adx_valid(self, target): if binary: subprocess.run([binary, "--config", str(target), "validate"], check=True) + @unittest.skipUnless(os.environ.get("AKERNEL_TEST_ADXCTL"), "requires packaged ADX CLI") + def test_packaged_cli_preserves_standalone_auto_capacity_after_preparation(self): + binary = os.environ["AKERNEL_TEST_ADXCTL"] + source = ROOT / "builder/config/adx-standalone.yaml" + + def resource_source(path): + deployment = yaml.safe_load(subprocess.run( + [binary, "--config", str(path), "config", "dump"], + check=True, capture_output=True, text=True, + ).stdout) + workers = [service for service in deployment["services"] + if service["role"] == "adxlet"] + self.assertEqual(len(workers), 1) + return workers[0]["config"]["resource_source"] + + baseline = resource_source(source) + self.assertEqual(baseline["kind"], "auto") + for pvm in (False, True): + for trace, mount in ((False, None), (True, None), + (False, Path("/mnt/akernel-shared"))): + with self.subTest(pvm=pvm, trace=trace, mount=mount), tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "prepared.yaml" + with patch.dict(os.environ, {"ENABLE_TRACE": str(trace).lower()}): + # Only mount inspection is simulated; capacity comes + # from the real packaged CLI's resolved deployments. + with patch.object(CONFIG.subprocess, "run") as run: + run.return_value.stdout = "afs-dfs fuse rw\n" + CONFIG.configure(source, target, mount, "test/checkpoints", pvm=pvm) + self.assertEqual(resource_source(target), baseline) + def test_trace_preserves_explicit_services_and_only_changes_present_roles(self): for profile in ("standalone-managed-redis", "node", "coordinator", "ingress-api"): source = ROOT / f"src/agent-dx/build/config/examples/deployment-{profile}.yaml" diff --git a/deploy/README.md b/deploy/README.md index 4473ace8..c5c1eb3f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,6 +17,91 @@ when a shared filesystem is independently required. For one deployment serving both native and E2B SDKs with version-independent application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-application-logs.md). +## Aggregate sandbox memory budget + +The direct Helm chart enables the cgroup v2 aggregate sandbox memory policy by +default for the ordinary KVM backend profile: + +```toml +[plugin.resource.memory_limit] +enabled = true +total_reserve = "16GiB" +``` + +Effective node memory is the minimum of host `MemTotal` and visible finite +ancestor cgroup limits. Sandboxd subtracts the reserve, rounds the result down +to a page boundary and preserves any existing stricter `/akernel/memory.max`. +The sandboxd resource socket is clipped to that effective parent ceiling, +without subtracting the reserve a second time. Helm and cloud Adxlet configs +consume that socket. For example, effective memory of 64 GiB and this reserve +produce a candidate of 48 GiB; an existing 40 GiB cap remains 40 GiB. + +Standalone keeps the policy disabled by default, with `total_reserve="16GiB"` +available for explicit configuration. Its bundled ADX profile retains the +`auto` resource source, including PVM, trace and shared-storage variants. +`auto` does not guarantee public scheduling capacity follows the sandbox parent +ceiling. Before enabling the policy, operators must configure a compatible ADX +`resource_source` and qualify memory capacity against the parent budget, +allocation and release after sandbox create/delete, and schedulable CPU. This +also applies to custom ADX configurations. The disabled default does not repair +or qualify resource-source behavior; CPU discovery is unchanged by this policy. + +Enabling the policy requires cgroup v2 and effective memory greater than the +reserve, with enough remaining capacity for the requested sandboxes. +Docker/OrbStack VM limits and visible ancestor limits participate in this +calculation. Use a matching node image built with the pinned sandboxd's +`total_reserve` support. On a smaller node, explicitly choose a smaller positive +`total_reserve` or keep `enabled=false` in the sandboxd TOML before starting. +Standalone reads its source config template; Helm uses +`node.config.sandboxd.config` (`core.node.config.sandboxd.config` in the umbrella +chart). Cloud profiles expose the enabled flag and reserve described below, +alongside their existing complete-config override. Values use integer bytes or +`B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not accepted. +Verify schedulable CPU from the selected resource provider as well as memory +before rollout, including when using Helm's sandboxd source. + +The PVM profile keeps this policy disabled by default because enabled parent +enforcement has not been qualified on its matching host. Selecting PVM overrides +the TOML `enabled` value through standalone +`AKERNEL_PVM_MEMORY_LIMIT_ENABLED=false` or Helm +`node.config.sandboxd.pvmMemoryLimitEnabled=false` (`core.` prefix in the umbrella +chart). Explicitly set that switch to `true` and choose a suitable positive +`total_reserve` in the source TOML or complete Helm config to opt in for host +qualification. Both switches accept only `true` or `false` and affect only PVM. +PVM CI retains the disabled policy and does not claim parent-limit coverage. + +Both cloud Terraform profiles keep the policy disabled by default. Their +`sandbox_memory_limit_enabled` input is an explicit opt-in, with +`sandbox_memory_total_reserve="16GiB"`. Guided profiles expose the same inputs: + +```bash +make config SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB +``` + +Select and qualify a cgroup v2 node image before enabling it: the current standard Aliyun +`node_pool_image_type="AliyunLinux3"` default uses cgroup v1, and Huawei's +selected node image also needs cgroup-mode verification. These options do not +change the node operating system or migrate existing nodes to cgroup v2. + +This reserve is deployment policy, not a measured component budget or a +guarantee against host OOM. Distill-fs retains its independent 64 GiB limit and +is not automatically deducted. Control components, other Pods and host kernel +memory remain outside the sandbox parent; their limits, reclaim behavior and +peak demand need separate consideration. In particular, kernel memory excluded +by a host's `cgroup.memory=nokmem` setting is not covered by the parent limit. + +For existing nodes, adding a finite cap to an unlimited parent or lowering a +cap requires an empty sandbox subtree and residual usage below the new limit. +An equal or stricter existing cap can be reused with live sandboxes; a larger +candidate never automatically raises it. Review the rollout before upgrading: +ConfigMap checksums trigger Pod replacement, and the current chart permits +200 unavailable nodes. Use a canary or a separately constrained rollout after +draining, rather than enabling the policy across live nodes together. Custom +TOML containing the removed reserve fields must be migrated even when the +policy is disabled. See the pinned sandboxd +[memory-limit guide](../src/sandboxd/doc/node-memory-limit.md) for migration and +the complete startup contract. + ## Guided deployment The image installs the complete gVisor bundle pinned by sandboxd's runtime manifest: runsc, the containerd shim, and adjacent `gvisor-bin/` helpers. Its SHA-512 verifies the archive, not a standalone runsc binary. Keep these executables together when building custom images; replacing only runsc can leave mismatched checkpoint/restore helpers. diff --git a/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl b/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl new file mode 100644 index 00000000..05c33bb2 --- /dev/null +++ b/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl @@ -0,0 +1,38 @@ +{{- $config := fromToml .config -}} +{{- if hasKey $config "Error" -}} +{{- fail "PVM sandboxd.config must be valid TOML with no duplicate fields" -}} +{{- end -}} +{{- $table := $config -}} +{{- range $key := list "plugin" "resource" "memory_limit" -}} +{{- /* sandboxd also accepts uppercase aliases for its struct fields. */ -}} +{{- if hasKey $table (upper $key) -}} +{{- fail "PVM memory policy requires lowercase plugin.resource.memory_limit keys" -}} +{{- end -}} +{{- if hasKey $table $key -}} +{{- $table = get $table $key -}} +{{- if not (kindIs "map" $table) -}} +{{- fail (printf "PVM memory policy requires %s to be a TOML table" $key) -}} +{{- end -}} +{{- else -}} +{{- $table = dict -}} +{{- end -}} +{{- end -}} +{{- if hasKey $table "enabled" -}} +{{- if not (kindIs "bool" (get $table "enabled")) -}} +{{- fail "PVM memory policy requires a boolean enabled field" -}} +{{- end -}} +{{- if ne (get $table "enabled") .enabled -}} +{{- $_ := set $table "enabled" .enabled -}} +{{- $rendered := toToml $config -}} +{{- if or (not $rendered) (hasKey (fromToml $rendered) "Error") -}} +{{- fail "Could not render the PVM sandboxd configuration as TOML" -}} +{{- end -}} +{{- $rendered -}} +{{- else -}} +{{- .config -}} +{{- end -}} +{{- else if .enabled -}} +{{- fail "PVM memory policy requires a boolean enabled field when opting in" -}} +{{- else -}} +{{- .config -}} +{{- end -}} diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index 97d2730b..33ab789d 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -57,6 +57,10 @@ data: {{- if not (has $firecrackerBackend (list "kvm" "pvm")) }} {{- fail "node.config.sandboxd.firecrackerBackend must be kvm or pvm" }} {{- end }} +{{- $pvmMemoryLimitEnabled := toString .Values.node.config.sandboxd.pvmMemoryLimitEnabled }} +{{- if not (has $pvmMemoryLimitEnabled (list "true" "false")) }} +{{- fail "node.config.sandboxd.pvmMemoryLimitEnabled must be true or false" }} +{{- end }} {{- if eq $firecrackerBackend "pvm" }} {{- if not (and (contains "[plugin.runtime.firecracker]" $sandboxdConfig) (contains "firecracker=\"/usr/local/bin/firecracker\"" $sandboxdConfig)) }} {{- fail "PVM selection requires the standard Firecracker config section and binary entry" }} @@ -83,6 +87,13 @@ data: {{- end }} {{- $sandboxdConfig = replace $chunkDbMarker (printf "chunk_db_size=%s" ($chunkDbSize | quote)) $sandboxdConfig }} {{- end }} +{{- if eq $firecrackerBackend "pvm" }} +{{- if not (semverCompare ">=3.17.0" .Capabilities.HelmVersion.Version) }} +{{- fail "PVM configuration requires Helm 3.17 or newer for TOML validation" }} +{{- end }} +{{- /* Parse after consuming template markers. Bind fromToml only on the PVM path. */}} +{{- $sandboxdConfig = tpl (.Files.Get "scripts/pvm-memory-limit.tpl") (dict "Template" .Template "config" $sandboxdConfig "enabled" (eq $pvmMemoryLimitEnabled "true")) }} +{{- end }} {{ $sandboxdConfig | indent 4 }} registry.json: | diff --git a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py index 01adb83e..29467266 100644 --- a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py +++ b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py @@ -4,7 +4,9 @@ from __future__ import annotations +import re import subprocess +import tempfile import tomllib import unittest from pathlib import Path @@ -14,10 +16,14 @@ CHART = Path(__file__).resolve().parents[1] -def render(*settings: str) -> dict[tuple[str, str], dict]: +def render( + *settings: str, set_files: dict[str, Path] | None = None +) -> dict[tuple[str, str], dict]: command = ["helm", "template", "akernel", str(CHART), "--namespace", "akernel-test"] for setting in settings: command.extend(["--set-string", setting]) + for key, path in (set_files or {}).items(): + command.extend(["--set-file", f"{key}={path}"]) result = subprocess.run(command, capture_output=True, text=True, check=True) return { (item["kind"], item["metadata"]["name"]): item @@ -27,6 +33,178 @@ def render(*settings: str) -> dict[tuple[str, str], dict]: class SharedCheckpointStorageTests(unittest.TestCase): + def assert_memory_config(self, config, enabled, reserve): + resource = config["plugin"]["resource"] + self.assertEqual( + resource["memory_limit"], + {"enabled": enabled, "total_reserve": reserve}, + ) + self.assertIs(resource["memory_limit"]["enabled"], enabled) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual(config["plugin"]["image"]["cgroup_memory_limit"], "64GiB") + + def test_default_parent_memory_limit_uses_16gib_total_reserve(self): + config = tomllib.loads( + render()[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + ) + self.assert_memory_config(config, True, "16GiB") + + def test_complete_config_override_preserves_disabled_and_small_node_policy(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for enabled, reserve in ((False, "16GiB"), (True, "2GiB")): + with self.subTest(enabled=enabled, reserve=reserve): + custom = default.replace( + "enabled = true", f"enabled = {str(enabled).lower()}", 1 + ).replace( + 'total_reserve = "16GiB"', f'total_reserve = "{reserve}"', 1 + ) + path.write_text(custom) + resources = render(set_files={"node.config.sandboxd.config": path}) + rendered = resources[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + self.assertEqual(rendered.rstrip(), custom.rstrip()) + self.assert_memory_config(tomllib.loads(rendered), enabled, reserve) + + def test_pvm_parent_limit_is_disabled_unless_explicitly_enabled(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "pvm-sandboxd.toml" + original = default.replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"', 1) + for key in ("enabled", '"enabled"', "'enabled'"): + for source_enabled in ("false", "true"): + path.write_text(original.replace("enabled = true", f"{key} = {source_enabled}", 1)) + for enabled in ("false", "true"): + with self.subTest(key=key, source_enabled=source_enabled, enabled=enabled): + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, enabled == "true", "2GiB") + self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) + resources = render("node.config.sandboxd.firecrackerBackend=pvm") + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, False, "16GiB") + + def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "kvm-sandboxd.toml" + path.write_text(default.replace("enabled = true", "enabled = false", 1)) + resources = render( + "node.config.sandboxd.pvmMemoryLimitEnabled=true", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, False, "16GiB") + + def test_pvm_parent_switch_accepts_equivalent_toml_policy_forms(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + header = "[plugin.resource.memory_limit]" + section = r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "pvm-sandboxd.toml" + for enabled, source in (("false", "true"), ("true", "false")): + canonical = default.replace("enabled = true", f"enabled = {source}", 1) + forms = { + "canonical": canonical, + "quoted": canonical.replace(header, '[plugin."resource".memory_limit]'), + "escaped": canonical.replace(header, r'[plugin."res\u006furce".memory_limit]'), + "inline": re.sub( + section, + f'memory_limit = {{ enabled = {source}, total_reserve = "16GiB" }}\n\n', + canonical, + ), + "dotted": re.sub( + section, + f'memory_limit.enabled = {source}\nmemory_limit.total_reserve = "16GiB"\n\n', + canonical, + ), + } + expected = None + for form, candidate in forms.items(): + with self.subTest(enabled=enabled, form=form): + self.assertEqual(tomllib.loads(candidate), tomllib.loads(canonical)) + path.write_text(candidate) + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, enabled == "true", "16GiB") + if expected is None: + expected = config + else: + self.assertEqual(config, expected) + + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + without_table = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "", default, + ) + without_enabled = default.replace("enabled = true\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for custom, has_table in ((without_table, False), (without_enabled, True)): + with self.subTest(has_table=has_table): + path.write_text(custom) + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + memory = config["plugin"]["resource"].get("memory_limit", {}) + self.assertFalse(memory.get("enabled", False)) + self.assertEqual("memory_limit" in config["plugin"]["resource"], has_table) + if has_table: + self.assertEqual(memory, {"total_reserve": "16GiB"}) + self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) + self.assertIn("firecracker-pvm", config["plugin"]["runtime"]["runtime_binary"]) + with self.assertRaises(subprocess.CalledProcessError) as error: + render( + "node.config.sandboxd.firecrackerBackend=pvm", + "node.config.sandboxd.pvmMemoryLimitEnabled=true", + set_files={"node.config.sandboxd.config": path}, + ) + self.assertIn("boolean enabled field when opting in", error.exception.stderr) + + def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for field in ('enabled = "true"', "enabled = 1", '"enabled" = "true"', "'enabled' = 1", + "enabled = true\nenabled = false", 'enabled = true\n"enabled" = false', + "enabled = true\n'enabled' = false"): + for opt_in in ("false", "true"): + with self.subTest(field=field, opt_in=opt_in): + path.write_text(default.replace("enabled = true", field, 1)) + with self.assertRaises(subprocess.CalledProcessError) as error: + render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={opt_in}", + set_files={"node.config.sandboxd.config": path}, + ) + self.assertRegex(error.exception.stderr, "boolean enabled field|duplicate fields") + + def test_pvm_rejects_invalid_memory_opt_in(self): + for value in ("1", "TRUE", "yes", ""): + with self.subTest(value=value): + with self.assertRaises(subprocess.CalledProcessError) as error: + render(f"node.config.sandboxd.pvmMemoryLimitEnabled={value}") + self.assertIn("pvmMemoryLimitEnabled must be true or false", error.exception.stderr) + def test_control_service_budgets_protect_voluntary_disruption(self): for replicas in (1, 3): resources = render(f"adx.ingressApi.replicas={replicas}") diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 156d83ab..eab9c69d 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -529,6 +529,9 @@ node: # pvm selects firecracker-pvm and removes Kata from this node profile. # The image must contain the validated PVM guest bundle. firecrackerBackend: kvm + # PVM parent memory enforcement requires separate host qualification. + # Explicit opt-in overrides memory_limit.enabled; set an appropriate reserve in config. + pvmMemoryLimitEnabled: false # Empty uses the distill-fs default (100 GiB on Linux). chunkDbSize: "" config: |- @@ -556,9 +559,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + enabled = true + # Requires effective memory above this reserve; tune for the deployment. + total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index 00c041cd..6ce52f59 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -228,6 +228,28 @@ configurations must carry the override themselves. The ordinary KVM profile is unchanged and the guest-agent startup budget remains 15 seconds; extending an RPC cleanup budget does not authorize increasing the guest startup deadline. +The aggregate sandbox memory policy remains disabled by default for PVM until +its finite-cap profile is qualified on the matching maintained host. Standalone +can explicitly select `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true`; Helm uses +`node.config.sandboxd.pvmMemoryLimitEnabled=true` (under `core` in the umbrella +chart). Both default to false and apply only to the PVM backend. Select a +`total_reserve` suitable for the actual L1 in its sandboxd configuration before +opting in, configure a compatible ADX `resource_source`, and qualify memory +capacity, allocation/release accounting and schedulable CPU on that host. The +bundled standalone profile retains `auto`, which does not guarantee capacity +follows the sandbox parent limit. Ordinary runsc tests and static profile +checks do not qualify this combination, and the 16 GiB deployment reserve is +not a universal host OOM safety budget. + +PVM configuration requires Helm 3.17 or newer for chart rendering, or Python +3.11 or newer for the standalone launcher. Standalone also accepts Python 3.10 +with `tomli` installed in the launcher's Python environment. The generators +validate the TOML before writing the configuration. Helm can override quoted +and inline tables. For standalone, use the standard +`[plugin.resource.memory_limit]` section with a boolean `enabled` field when +the switch needs to change that value. Standalone rejects an override it +cannot apply and leaves the previous generated configuration in place. + ## Isolated qualification workflow Ordinary sandboxd hardware-KVM CI and AKernel's runsc-only standalone image do diff --git a/deploy/pvm/run-adx-ci.sh b/deploy/pvm/run-adx-ci.sh index 80e03220..04433dd7 100644 --- a/deploy/pvm/run-adx-ci.sh +++ b/deploy/pvm/run-adx-ci.sh @@ -64,6 +64,9 @@ for name,section,key in [('firecracker','vmm','binary_sha256'),('vmlinux','guest PAYLOAD docker image inspect --format '{{.Id}}' "$IMAGE" > "${out}/image-id.txt" export AKERNEL_FIRECRACKER_BACKEND=pvm AKERNEL_CHUNK_DB_SIZE=1GiB +# Aggregate parent enforcement remains outside the qualified PVM profile. +export AKERNEL_PVM_MEMORY_LIMIT_ENABLED=false +unset AKERNEL_TEST_EXPECT_MEMORY_BYTES AKERNEL_TEST_EXPECT_CPU_MILLIS export AKERNEL_CONTAINER_PROXY=http://10.0.2.2:3128 export AKERNEL_CONTAINER_NO_PROXY=localhost,127.0.0.1,172.17.0.1,10.88.0.1 # Keep Redis AOF/control writes on the L1 root disk while snapshot writes use XFS. @@ -144,7 +147,7 @@ export AKERNEL_TEST_CHECKPOINT_MEMORY_MIB=2048 python3 - "$out" <<'SETTINGS' import json,sys from pathlib import Path -(Path(sys.argv[1])/'qualification-settings.json').write_text(json.dumps({'runtime':'firecracker-pvm','virtiofs_enabled':True,'checkpoint_memory_mib':2048,'other_sdk_memory_mib':2048,'checkpoint_snapshot_type':'Full (requested explicitly by ADX)','debugctl_applied':False,'dedicated_pvm_profile':True},indent=2)) +(Path(sys.argv[1])/'qualification-settings.json').write_text(json.dumps({'runtime':'firecracker-pvm','virtiofs_enabled':True,'checkpoint_memory_mib':2048,'other_sdk_memory_mib':2048,'checkpoint_snapshot_type':'Full (requested explicitly by ADX)','debugctl_applied':False,'dedicated_pvm_profile':True,'sandbox_parent_memory_limit_enabled':False,'sandbox_parent_memory_limit_qualified':False},indent=2)) SETTINGS export PYTHONPATH="${root}/sdk/python" export AKERNEL_TEST_IMAGE=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["oci"])' "$fixtures") diff --git a/deploy/pvm/test_host_profile.py b/deploy/pvm/test_host_profile.py index 0f9aea7a..fd9ac3fa 100644 --- a/deploy/pvm/test_host_profile.py +++ b/deploy/pvm/test_host_profile.py @@ -2,8 +2,10 @@ """Check the maintained profile and Bash compatibility exports without builds.""" import hashlib import json +import os from pathlib import Path import subprocess +import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] @@ -45,5 +47,31 @@ def test_shell_exports_are_derived_from_the_json_profile(self): self.assertEqual(result.stdout.splitlines(), list(expected.values())) +class CiMemoryPolicyTest(unittest.TestCase): + def test_pvm_ci_keeps_parent_enforcement_disabled_even_if_inherited_enabled(self): + script = (ROOT / "deploy/pvm/run-adx-ci.sh").read_text() + preparation = "export AKERNEL_FIRECRACKER_BACKEND=pvm" + script.split( + "export AKERNEL_FIRECRACKER_BACKEND=pvm", 1 + )[1].split("export AKERNEL_CONTAINER_PROXY=", 1)[0] + result = subprocess.run( + ["bash", "-euc", preparation + '\npython3 -c \'import os; print(os.environ["AKERNEL_PVM_MEMORY_LIMIT_ENABLED"]); assert "AKERNEL_TEST_EXPECT_MEMORY_BYTES" not in os.environ; assert "AKERNEL_TEST_EXPECT_CPU_MILLIS" not in os.environ\''], + text=True, capture_output=True, + env=dict(os.environ, AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true", + AKERNEL_TEST_EXPECT_MEMORY_BYTES="6442450944", + AKERNEL_TEST_EXPECT_CPU_MILLIS="1000"), timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "false") + + def test_pvm_qualification_settings_do_not_claim_parent_limit_coverage(self): + script = (ROOT / "deploy/pvm/run-adx-ci.sh").read_text() + settings = script.split("<<'SETTINGS'\n", 1)[1].split("\nSETTINGS", 1)[0] + with tempfile.TemporaryDirectory() as directory: + subprocess.run(["python3", "-c", settings, directory], check=True, timeout=10) + recorded = json.loads((Path(directory) / "qualification-settings.json").read_text()) + self.assertFalse(recorded["sandbox_parent_memory_limit_enabled"]) + self.assertFalse(recorded["sandbox_parent_memory_limit_qualified"]) + + if __name__ == "__main__": unittest.main() diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index f7da40dc..6326f35e 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -32,6 +32,8 @@ install_monitor_override="" install_dragonfly_override="" node_image_pull_mode_override="" chunk_db_size_override="" +sandbox_memory_limit_enabled_override="" +sandbox_memory_total_reserve_override="" juicefs_enabled="false" juicefs_storage_class="" juicefs_cache_mib="4096" @@ -134,6 +136,14 @@ while [[ $# -gt 0 ]]; do chunk_db_size_override="$2" shift 2 ;; + --sandbox-memory-limit-enabled) + sandbox_memory_limit_enabled_override="$2" + shift 2 + ;; + --sandbox-memory-total-reserve) + sandbox_memory_total_reserve_override="$2" + shift 2 + ;; --juicefs-enabled) juicefs_enabled="$2" shift 2 @@ -334,6 +344,15 @@ set_or_prompt chunk_db_size "Shared ChunkDB capacity (empty uses distill-fs defa if [[ -n "${chunk_db_size}" && ! "${chunk_db_size}" =~ ^[0-9]+(B|KiB|MiB|GiB|TiB)?$ ]]; then die "CHUNK_DB_SIZE must be whole bytes or an integer with B/KiB/MiB/GiB/TiB" fi +set_or_prompt sandbox_memory_limit_enabled \ + "Enable aggregate sandbox memory limit on qualified cgroup v2 nodes (true/false)" \ + "false" "${sandbox_memory_limit_enabled_override}" +set_or_prompt sandbox_memory_total_reserve \ + "Aggregate sandbox memory reserve (effective memory must exceed this when enabled)" \ + "16GiB" "${sandbox_memory_total_reserve_override}" +if [[ ! "${sandbox_memory_total_reserve}" =~ ^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$ ]]; then + die "SANDBOX_MEMORY_TOTAL_RESERVE must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB" +fi set_or_prompt enable_runc "Enable the optional runc runtime (true/false)" "false" "${enable_runc_override}" set_or_prompt schedule_placement_policy \ "YuanRong schedule placement policy (binpack/spread)" "spread" \ @@ -347,6 +366,7 @@ if [[ -z "${grafana_admin_password}" ]]; then fi install_monitor="$(normalize_bool "${install_monitor}")" install_dragonfly="$(normalize_bool "${install_dragonfly}")" +sandbox_memory_limit_enabled="$(normalize_bool "${sandbox_memory_limit_enabled}")" enable_runc="$(normalize_bool "${enable_runc}")" schedule_placement_policy="$(normalize_schedule_placement_policy "${schedule_placement_policy}")" grafana_public_access="$(normalize_bool "${grafana_public_access}")" @@ -444,6 +464,8 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} chunk_db_size = "${chunk_db_size}" +sandbox_memory_limit_enabled = ${sandbox_memory_limit_enabled} +sandbox_memory_total_reserve = "${sandbox_memory_total_reserve}" enable_runc = ${enable_runc} schedule_placement_policy = "${schedule_placement_policy}" EOF @@ -490,6 +512,8 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} chunk_db_size = "${chunk_db_size}" +sandbox_memory_limit_enabled = ${sandbox_memory_limit_enabled} +sandbox_memory_total_reserve = "${sandbox_memory_total_reserve}" enable_runc = ${enable_runc} schedule_placement_policy = "${schedule_placement_policy}" EOF @@ -514,6 +538,8 @@ MONITOR_NAMESPACE=akernel-monitor INSTALL_DRAGONFLY=${install_dragonfly} NODE_IMAGE_PULL_MODE=${node_image_pull_mode} CHUNK_DB_SIZE=${chunk_db_size} +SANDBOX_MEMORY_LIMIT_ENABLED=${sandbox_memory_limit_enabled} +SANDBOX_MEMORY_TOTAL_RESERVE=${sandbox_memory_total_reserve} AKERNEL_ENABLE_RUNC=${enable_runc} SCHEDULE_PLACEMENT_POLICY=${schedule_placement_policy} EOF diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 6dc8c9f8..84ec6995 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -58,6 +58,39 @@ for inspection and retry. Data and credentials survive stopping. For an independent data profile, set an absolute `AKERNEL_STANDALONE_DATA_DIR` for every command. Only one standalone node can run per Docker daemon. +Standalone keeps the aggregate sandbox memory limit disabled by default, with +`total_reserve="16GiB"` available for explicit configuration. To enable it, edit +`[plugin.resource.memory_limit]` in +`deploy/standalone/config/sandboxd_config.toml` before startup. Qualify cgroup v2 +and effective memory above the selected positive reserve, including any +Docker/OrbStack VM or visible ancestor limit, with room for sandbox requests. +Editing only the generated data-directory TOML is insufficient because startup +regenerates it. This also applies to `make standalone-source-test`. See +[the memory budget and upgrade contract](../README.md#aggregate-sandbox-memory-budget) +before enabling the policy on an existing profile. + +Selecting `AKERNEL_FIRECRACKER_BACKEND=pvm` keeps the parent limit disabled by +default, including when the source TOML has `enabled=true`. PVM parent-limit +enforcement still needs qualification on the matching host. To opt in for that +qualification, set `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` and choose a suitable +positive `total_reserve` in the source TOML before startup. The switch accepts +only `true` or `false`, defaults to `false`, and affects only PVM. The maintained +PVM CI fixture keeps it disabled. + +For PVM, use Python 3.11 or newer, or install `tomli` in the Python 3.10 +environment used by the launcher. To override a custom memory policy, put its +boolean `enabled` field in the standard `[plugin.resource.memory_limit]` +section. See the [PVM deployment guide](../pvm-runtime.md#deployment-and-storage-profile). + +The bundled ADX profile retains its `auto` resource source, including PVM and +trace/shared-storage variants. `auto` does not guarantee that public scheduling +capacity follows the sandbox parent ceiling. Before enabling the policy, +operators must configure and qualify a compatible ADX `resource_source`, whether +using the bundled profile or a custom `AKERNEL_ADX_CONFIG`. Verify memory +capacity against the parent budget, allocation and release after sandbox +create/delete, and schedulable CPU from that source. The disabled default does +not repair or qualify resource-source behavior. + ### Runtime selection Runsc is the source default. Opt into runc with diff --git a/deploy/standalone/config/sandboxd_config.toml b/deploy/standalone/config/sandboxd_config.toml index 240161e3..c6c0fea2 100644 --- a/deploy/standalone/config/sandboxd_config.toml +++ b/deploy/standalone/config/sandboxd_config.toml @@ -30,8 +30,8 @@ enable_cache_in_stock_mode=true [plugin.resource.memory_limit] enabled = false -# Set a deployment-specific total reserve before enabling. -# total_reserve = "22GiB" +# Before enabling, qualify scheduling capacity and tune this deployment reserve. +total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/standalone/prepare-pvm-memory.py b/deploy/standalone/prepare-pvm-memory.py new file mode 100644 index 00000000..b2dfd9de --- /dev/null +++ b/deploy/standalone/prepare-pvm-memory.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Apply the standalone PVM switch to a validated sandboxd configuration.""" + +import argparse +from pathlib import Path +import re +import sys + +try: + import tomllib +except ModuleNotFoundError: + try: + import tomli as tomllib + except ModuleNotFoundError: + raise SystemExit("PVM configuration requires Python 3.11+ or tomli on Python 3.10") from None + + +MEMORY_TABLE = re.compile( + r"(?ms)^[\t ]*\[plugin\.resource\.memory_limit\][\t ]*(?:#[^\n]*)?\n" + r"(?P.*?)(?=^[\t ]*\[|\Z)" +) +ENABLED = re.compile( + r"""(?m)^([\t ]*)(?:enabled|"enabled"|'enabled')[\t ]*=[\t ]*""" + r"(?:true|false)(?=[\t ]*(?:#|$))" +) + + +def configure(source: str, enabled: bool) -> str: + config = tomllib.loads(source) + table = config + for key in ("plugin", "resource", "memory_limit"): + # Match the sandboxd decoder's acceptance of uppercase struct aliases. + if key.upper() in table: + raise ValueError("PVM memory policy requires lowercase plugin.resource.memory_limit keys") + table = table.get(key, {}) + if not isinstance(table, dict): + raise ValueError(f"PVM memory policy requires {key} to be a TOML table") + + if "enabled" not in table: + if enabled: + raise ValueError("PVM memory policy requires a boolean enabled field when opting in") + return source + current = table["enabled"] + if not isinstance(current, bool): + raise ValueError("PVM memory policy requires a boolean enabled field") + if current == enabled: + return source + + # Verify the full parsed result so a lookalike line in a string cannot + # satisfy the override or change unrelated fields. + table["enabled"] = enabled + for match in MEMORY_TABLE.finditer(source): + body = ENABLED.sub(r"\1enabled = " + str(enabled).lower(), match["body"]) + rendered = source[:match.start("body")] + body + source[match.end("body"):] + if tomllib.loads(rendered) == config: + return rendered + raise ValueError( + "Use [plugin.resource.memory_limit] with a boolean enabled field " + "to override the PVM memory policy" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path) + parser.add_argument("enabled", choices=("true", "false")) + args = parser.parse_args() + try: + sys.stdout.write(configure(args.config.read_text(), args.enabled == "true")) + except tomllib.TOMLDecodeError: + parser.exit(1, "PVM sandboxd configuration must be valid TOML with no duplicate fields\n") + except ValueError as error: + parser.exit(1, f"{error}\n") + + +if __name__ == "__main__": + main() diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 7dfe1b8e..bc30c74b 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -24,6 +24,7 @@ AKERNEL_DATA_PORT="${AKERNEL_DATA_PORT:-80}" AKERNEL_ENDPOINT_HOST="${AKERNEL_ENDPOINT_HOST:-127.0.0.1}" AKERNEL_CHUNK_DB_SIZE="${AKERNEL_CHUNK_DB_SIZE:-}" AKERNEL_FIRECRACKER_BACKEND="${AKERNEL_FIRECRACKER_BACKEND:-kvm}" +AKERNEL_PVM_MEMORY_LIMIT_ENABLED="${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" # Container runtime command (docker or pouch) DOCKER_CMD="" @@ -191,6 +192,17 @@ check_prerequisites() { exit 1 ;; esac + case "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED}" in + true|false) ;; + *) + log_error "AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false" + exit 1 + ;; + esac + if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]] && + ! python3 "${SCRIPT_DIR}/prepare-pvm-memory.py" --help >/dev/null; then + exit 1 + fi if [[ "${DATA_DIR}" != /* ]]; then log_error "AKERNEL_STANDALONE_DATA_DIR must be an absolute path" exit 1 @@ -317,12 +329,20 @@ configure_gpu() { configure_network() { local config_tmp="${SANDBOXD_CONFIG_FILE}.tmp" + local pvm_config local sed_args=( -E -e "s/^[[:space:]]*nat_backend[[:space:]]*=.*/nat_backend=\"${AKERNEL_NAT_BACKEND}\"/" ) if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]]; then + case "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" in + true|false) ;; + *) + log_error "AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false" + exit 1 + ;; + esac if ! grep -q '^\[plugin.runtime.firecracker\]$' \ "${CONFIG_DIR}/sandboxd_config.toml" || \ ! grep -q '^firecracker="/usr/local/bin/firecracker"$' \ @@ -330,6 +350,8 @@ configure_network() { log_error "PVM selection requires the standard Firecracker config section and binary entry" exit 1 fi + pvm_config="$(python3 "${SCRIPT_DIR}/prepare-pvm-memory.py" \ + "${CONFIG_DIR}/sandboxd_config.toml" "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}")" || exit 1 sed_args+=( -e 's/^firecracker=/firecracker-pvm=/' -e 's/^\[plugin.runtime.firecracker\]$/[plugin.runtime.firecracker_pvm]/' @@ -396,7 +418,11 @@ configure_network() { -e "s|^[[:space:]]*# AKERNEL_CHUNK_DB_SIZE[[:space:]]*$|chunk_db_size=\"${AKERNEL_CHUNK_DB_SIZE}\"|" ) fi - sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" + if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]]; then + sed "${sed_args[@]}" <<< "${pvm_config}" > "${config_tmp}" + else + sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" + fi mv "${config_tmp}" "${SANDBOXD_CONFIG_FILE}" if [[ "${AKERNEL_ENABLE_RUNC}" == "true" ]]; then diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index b936fb1d..4e8f0646 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -11,6 +11,11 @@ import tempfile import unittest +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + STANDALONE_DIR = Path(__file__).resolve().parent DIRECT_MARKER = "# AKERNEL_DIRECT_RESOLVER" @@ -57,6 +62,7 @@ def generate(self, **overrides): AKERNEL_NAT_BACKEND="iptables", AKERNEL_ENABLE_RUNC="true", AKERNEL_FIRECRACKER_BACKEND="kvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="false", AKERNEL_RUNC_RESOLV_CONF=str(self.source), AKERNEL_CHUNK_DB_SIZE="", ) @@ -93,6 +99,39 @@ def assert_rejected_without_replacing_config(self, message): self.assertEqual(self.config_output.read_text(), "existing config\n") self.assertFalse(self.resolver_output.exists()) + def assert_memory_config(self, enabled, reserve): + config = self.read_sections() + policy = tomllib.loads(self.config_output.read_text())["plugin"]["resource"]["memory_limit"] + self.assertEqual(policy, {"enabled": enabled == "true", "total_reserve": reserve}) + self.assertIs(policy["enabled"], enabled == "true") + self.assertEqual(config["plugin.resource"]["cgroup_root_name"], '"/akernel"') + self.assertEqual(config["plugin.resource"]["max_instance_num"], "1000") + self.assertEqual( + config["plugin.resource"]["enable_cache_in_stock_mode"], "true" + ) + self.assertEqual(config["plugin.image"]["cgroup_memory_limit"], '"64GiB"') + + def test_generated_config_disables_parent_limit_with_16gib_reserve(self): + result = self.generate(AKERNEL_ENABLE_RUNC="false") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config("false", "16GiB") + + def test_generated_config_preserves_explicit_memory_policy_overrides(self): + original = self.template.read_text() + for enabled, reserve in (("false", "16GiB"), ("true", "2GiB")): + with self.subTest(enabled=enabled, reserve=reserve): + custom, count = re.subn( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "[plugin.resource.memory_limit]\n" + f"enabled = {enabled}\ntotal_reserve = \"{reserve}\"\n\n", + original, + ) + self.assertEqual(count, 1) + self.template.write_text(custom) + result = self.generate(AKERNEL_ENABLE_RUNC="false") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, reserve) + def test_pvm_selection_preserves_direct_resolver_configuration(self): result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") self.assertEqual(result.returncode, 0, result.stderr) @@ -103,6 +142,124 @@ def test_pvm_selection_preserves_direct_resolver_configuration(self): self.assertNotIn("firecracker", config["plugin.runtime.runtime_binary"]) self.assertNotIn("kata", config["plugin.runtime.runtime_binary"]) self.assertEqual(self.resolver_output.read_text(), self.resolver_content) + self.assert_memory_config("false", "16GiB") + + def test_pvm_parent_limit_requires_explicit_opt_in_and_preserves_reserve(self): + original = self.template.read_text().replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"') + for key in ("enabled", '"enabled"', "'enabled'"): + for source_enabled in ("false", "true"): + self.template.write_text(original.replace("enabled = false", f"{key} = {source_enabled}", 1)) + for enabled in ("false", "true"): + with self.subTest(key=key, source_enabled=source_enabled, enabled=enabled): + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, "2GiB") + + def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): + original = self.template.read_text() + for enabled in ("false", "true"): + with self.subTest(enabled=enabled): + self.template.write_text(original.replace("enabled = false", f"enabled = {enabled}", 1)) + result = self.generate(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, "16GiB") + + def test_pvm_rejects_policy_rewrites_without_a_standard_memory_table(self): + original = self.template.read_text() + header = "[plugin.resource.memory_limit]" + for form in ("quoted", "inline", "escaped", "dotted"): + for enabled, source in (("false", "true"), ("true", "false")): + with self.subTest(form=form, enabled=enabled): + canonical = original.replace("enabled = false", f"enabled = {source}", 1) + if form == "quoted": + custom = canonical.replace(header, '[plugin."resource".memory_limit]') + elif form == "escaped": + custom = canonical.replace(header, r'[plugin."res\u006furce".memory_limit]') + else: + if form == "inline": + policy = f'memory_limit = {{ enabled = {source}, total_reserve = "16GiB" }}' + else: + policy = f'memory_limit.enabled = {source}\nmemory_limit.total_reserve = "16GiB"' + custom = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + policy + "\n\n", canonical, + ) + self.assertEqual(tomllib.loads(custom), tomllib.loads(canonical)) + self.template.write_text(custom) + self.config_output.write_text("existing config\n") + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use [plugin.resource.memory_limit]", result.stderr) + self.assertEqual(self.config_output.read_text(), "existing config\n") + self.assertFalse(self.resolver_output.exists()) + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=source, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(source, "16GiB") + self.resolver_output.unlink() + + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): + original = self.template.read_text() + without_table = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "", original, + ) + without_enabled = original.replace("enabled = false\n", "", 1) + for custom, has_table in ((without_table, False), (without_enabled, True)): + with self.subTest(has_table=has_table): + self.template.write_text(custom) + result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") + self.assertEqual(result.returncode, 0, result.stderr) + config = self.read_sections() + self.assertFalse(config.getboolean("plugin.resource.memory_limit", "enabled", fallback=False)) + self.assertEqual(config.has_section("plugin.resource.memory_limit"), has_table) + if has_table: + self.assertEqual(dict(config["plugin.resource.memory_limit"]), {"total_reserve": '"16GiB"'}) + self.assertIn("plugin.runtime.firecracker_pvm", config) + self.assertIn("firecracker-pvm", config["plugin.runtime.runtime_binary"]) + generated = self.config_output.read_text() + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true", + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("boolean enabled field when opting in", result.stderr) + self.assertEqual(self.config_output.read_text(), generated) + + def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): + original = self.template.read_text() + for field in ('enabled = "true"', "enabled = 1", '"enabled" = "true"', "'enabled' = 1", + "enabled = true\nenabled = false", 'enabled = true\n"enabled" = false', + "enabled = true\n'enabled' = false"): + for opt_in in ("false", "true"): + with self.subTest(field=field, opt_in=opt_in): + self.template.write_text(original.replace("enabled = false", field, 1)) + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=opt_in, + ) + self.assertNotEqual(result.returncode, 0) + self.assertRegex(result.stderr, "boolean enabled field|duplicate fields") + self.assertFalse(self.config_output.exists()) + + def test_pvm_rejects_invalid_memory_opt_in(self): + for value in ("", "1", "TRUE", "yes", "true; false"): + with self.subTest(value=value): + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=value, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false", result.stderr) + self.assertFalse(self.config_output.exists()) def test_direct_source_is_under_runtime_and_node_source_is_preserved(self): for backend in ("iptables", "bpfnat"): diff --git a/deploy/standalone/test_start_platform.py b/deploy/standalone/test_start_platform.py index 3126eb4c..683f0d16 100644 --- a/deploy/standalone/test_start_platform.py +++ b/deploy/standalone/test_start_platform.py @@ -57,6 +57,7 @@ def setUp(self): YR_IMAGE_PROCESS_CONFIG="/run/akernel/yr-image-process.json", AKERNEL_ENABLE_RUNC="true", AKERNEL_NAT_BACKEND="iptables", AKERNEL_FIRECRACKER_BACKEND="kvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="false", MOCK_HOST="Darwin", MOCK_OS="OrbStack", MOCK_ENGINE="linux/aarch64", MOCK_IMAGE="linux/arm64", MOCK_TRAEFIK_IMAGE="linux/arm64", @@ -176,6 +177,12 @@ def test_invalid_firecracker_backend_is_rejected_before_profile_mutation(self): self.assertIn("AKERNEL_FIRECRACKER_BACKEND must be kvm or pvm", result.stderr) self.assert_no_profile_mutation() + def test_invalid_pvm_memory_opt_in_is_rejected_before_profile_mutation(self): + result = self.run_main(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="yes") + self.assertNotEqual(result.returncode, 0) + self.assertIn("AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false", result.stderr) + self.assert_no_profile_mutation() + def test_native_runc_uses_only_disposable_capability_probe(self): result = self.run_script( "validate_host_platform\nvalidate_image_capabilities\nprepare_host_network_modules" diff --git a/deploy/terraform/aliyun/README.md b/deploy/terraform/aliyun/README.md index 2c9f24d6..219e7177 100644 --- a/deploy/terraform/aliyun/README.md +++ b/deploy/terraform/aliyun/README.md @@ -10,6 +10,27 @@ It installs: - Optional Dragonfly to `dragonfly_namespace` (controlled by `install_dragonfly`) - Optional prereq `openkruise` to `prereq_namespace` (controlled by `install_prereqs`) +## Aggregate sandbox memory budget + +The cloud profile leaves the parent memory policy disabled by default and +retains its existing OS selection. Standard `AliyunLinux3` uses cgroup v1; +the policy requires cgroup v2. Before opting in, select and qualify a v2 node +image, verify the actual node/container cgroup mode, and size the reserve for +that profile. The module does not convert an existing node's cgroup mode. + +Set `sandbox_memory_limit_enabled=true` and +`sandbox_memory_total_reserve="16GiB"` in `terraform.tfvars`, or pass +`SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB` to +`make config`. Effective memory, including finite ancestors, must exceed the +reserve and leave room for sandbox requests. Smaller nodes require an explicit +smaller positive reserve. These settings also apply when reusing a cluster; +complete TOML overrides remain responsible for their own memory policy. + +Adding or lowering a parent cap requires drained sandboxes. The reserve is +deployment policy rather than a host-OOM guarantee; distill-fs has an independent +limit. See [the memory budget and upgrade contract](../../README.md#aggregate-sandbox-memory-budget) +before enabling the policy on existing nodes. + ## Pod PID budget `node_pool_pids_limit` defaults to `1620780` for the default pool and all diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index c6cf879e..c3bfb9aa 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -176,6 +176,9 @@ locals { core_chart_hash = sha256(join("", [for f in fileset("${path.module}/../../akernel/charts/core", "**") : filesha256("${path.module}/../../akernel/charts/core/${f}")])) core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { + sandbox_memory_limit_enabled = var.sandbox_memory_limit_enabled + sandbox_memory_total_reserve = var.sandbox_memory_total_reserve + cluster_name = var.cluster_name adx_public_service_enabled = var.adx_public_service_enabled adx_service_type = var.adx_service_type diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index bb39c5e2..80d9284d 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -151,6 +151,11 @@ node_secret_create = true # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Explicit opt-in after qualifying cgroup v2 and the effective memory budget. +# Standard AliyunLinux3 defaults to cgroup v1; select a qualified v2 node image. +sandbox_memory_limit_enabled = false +sandbox_memory_total_reserve = "16GiB" + # AKernel defaults YuanRong placement to "spread"; use "binpack" to compact. schedule_placement_policy = "spread" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 89df148a..1539b160 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -173,9 +173,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + # Explicit opt-in requires qualified cgroup v2 nodes and sufficient memory. + enabled = ${sandbox_memory_limit_enabled} + total_reserve = ${jsonencode(sandbox_memory_total_reserve)} [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index 2eb76daf..32141473 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -834,6 +834,22 @@ variable "tempo_resources" { default = {} } +variable "sandbox_memory_limit_enabled" { + type = bool + description = "Opt into the aggregate sandbox parent memory limit only on qualified cgroup v2 nodes. Drain before adding or lowering a cap." + default = false +} + +variable "sandbox_memory_total_reserve" { + type = string + description = "Memory excluded from the aggregate sandbox budget when enabled; effective node memory must exceed this deployment-specific reserve." + default = "16GiB" + validation { + condition = can(regex("^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$", var.sandbox_memory_total_reserve)) + error_message = "sandbox_memory_total_reserve must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB." + } +} + variable "chunk_db_size" { type = string description = "Shared distill-fs ChunkDB map capacity, e.g. 64GiB; empty uses the Linux default of 100 GiB. Existing caches must not be resized." diff --git a/deploy/terraform/huaweicloud/README.md b/deploy/terraform/huaweicloud/README.md index a6d5bd43..784cbdf6 100644 --- a/deploy/terraform/huaweicloud/README.md +++ b/deploy/terraform/huaweicloud/README.md @@ -6,6 +6,27 @@ contract as the Aliyun module: one all-in-one AKernel image, an Agent DX identity Secret, managed Redis, a direct ADX public Service, optional public Grafana, and local state under `.akernel//`. +## Aggregate sandbox memory budget + +The cloud profile leaves the parent memory policy disabled by default and +retains its existing OS selection. Before opting in, qualify cgroup v2 on the +actual CCE node image and inside the AKernel container, then size the reserve +for that profile. The OS name alone does not establish the cgroup mode, and +the module does not convert existing nodes to cgroup v2. + +Set `sandbox_memory_limit_enabled=true` and +`sandbox_memory_total_reserve="16GiB"` in `terraform.tfvars`, or pass +`SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB` to +`make config`. Effective memory, including finite ancestors, must exceed the +reserve and leave room for sandbox requests. Smaller nodes require an explicit +smaller positive reserve. These settings also apply when reusing a cluster; +complete TOML overrides remain responsible for their own memory policy. + +Adding or lowering a parent cap requires drained sandboxes. The reserve is +deployment policy rather than a host-OOM guarantee; distill-fs has an independent +limit. See [the memory budget and upgrade contract](../../README.md#aggregate-sandbox-memory-budget) +before enabling the policy on existing nodes. + ## Pod PID budget [CCE documents `pod-pids-limit=-1` by default](https://support.huaweicloud.com/intl/en-us/usermanual-cce/cce_10_0652.html), diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index 2626e66a..39c227a4 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -87,6 +87,9 @@ locals { ]) : local.node_pool_bootstrap_script core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { + sandbox_memory_limit_enabled = var.sandbox_memory_limit_enabled + sandbox_memory_total_reserve = var.sandbox_memory_total_reserve + cluster_name = var.cluster_name adx_public_service_enabled = var.adx_public_service_enabled adx_service_type = var.adx_service_type diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index 7fff9e24..7b4de2b0 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -88,6 +88,11 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Explicit opt-in after qualifying the actual node image's cgroup v2 mode +# and effective memory budget; the OS name alone does not establish support. +sandbox_memory_limit_enabled = false +sandbox_memory_total_reserve = "16GiB" + # AKernel defaults YuanRong placement to "spread"; use "binpack" to compact. schedule_placement_policy = "spread" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 031b4209..6696a174 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -142,9 +142,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + # Explicit opt-in requires qualified cgroup v2 nodes and sufficient memory. + enabled = ${sandbox_memory_limit_enabled} + total_reserve = ${jsonencode(sandbox_memory_total_reserve)} [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index bf2e29f1..655a2d75 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -830,6 +830,22 @@ variable "tempo_resources" { default = {} } +variable "sandbox_memory_limit_enabled" { + type = bool + description = "Opt into the aggregate sandbox parent memory limit only on qualified cgroup v2 nodes. Drain before adding or lowering a cap." + default = false +} + +variable "sandbox_memory_total_reserve" { + type = string + description = "Memory excluded from the aggregate sandbox budget when enabled; effective node memory must exceed this deployment-specific reserve." + default = "16GiB" + validation { + condition = can(regex("^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$", var.sandbox_memory_total_reserve)) + error_message = "sandbox_memory_total_reserve must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB." + } +} + variable "chunk_db_size" { type = string description = "Shared distill-fs ChunkDB map capacity, e.g. 64GiB; empty uses the Linux default of 100 GiB. Existing caches must not be resized." diff --git a/deploy/terraform/test_adx_contract.py b/deploy/terraform/test_adx_contract.py index 080f5c2b..d46e0d3e 100755 --- a/deploy/terraform/test_adx_contract.py +++ b/deploy/terraform/test_adx_contract.py @@ -5,14 +5,180 @@ from __future__ import annotations import json +import os import re import shutil import subprocess import tempfile +import textwrap import unittest from pathlib import Path ROOT = Path(__file__).resolve().parent +MEMORY_INPUTS = ("sandbox_memory_limit_enabled", "sandbox_memory_total_reserve") + + +def sandboxd_config(values): + # ACK's Python 3.10 tests share evaluate_fixture without parsing TOML. + import tomllib + + block = re.search(r"(?ms)^ config: \|-\n(.*?)(?=^ {0,6}\S|\Z)", values) + if block is None: + raise AssertionError("sandboxd TOML missing from generated cloud values") + return tomllib.loads(textwrap.dedent(block.group(1))) + + +@unittest.skipUnless(shutil.which("terraform"), "Terraform is required for template evaluation") +class SandboxMemoryPolicyValidationTest(unittest.TestCase): + def render(self, provider, inputs): + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) + variables = (ROOT / provider / "variables.tf").read_text() + blocks = [ + re.search(rf'(?ms)^variable "{name}" {{.*?^}}', variables).group() + for name in MEMORY_INPUTS + ] + main = (ROOT / provider / "main.tf").read_text() + mapping = re.search( + r'(?ms)^ core_values = templatefile\("\$\{path.module\}/values-akernel.yaml.tmpl", \{.*?^ \}\)', + main, + ).group() + # Keep the production templatefile call and memory-input wiring. + # Other deployment inputs use inert fixtures: no cloud providers, + # credentials, node pools or Helm releases participate in the plan. + booleans = { + "acr_secret_enabled", "enable_kruise", "install_monitor", + "adx_public_service_enabled", "enable_runc", "native_filestore", + "node_home_use_csi_ephemeral", "node_secret_create", "node_image_pull_auto", + } + + def fixture_input(match): + key = match.group(1) + if key in MEMORY_INPUTS: + return match.group() + value = False if key in booleans else "" + return f" {key} = {json.dumps(value)}" + + mapping = re.sub(r"(?m)^ (\w+)\s+= .*?$", fixture_input, mapping) + blocks.append("locals {\n" + mapping + "\n}") + blocks.append('output "rendered" { value = local.core_values }') + (fixture / "main.tf").write_text("\n".join(blocks)) + shutil.copy2(ROOT / provider / "values-akernel.yaml.tmpl", fixture) + (fixture / "inputs.tfvars.json").write_text(json.dumps(inputs)) + subprocess.run( + ["terraform", "init", "-backend=false", "-input=false", "-no-color"], + cwd=fixture, capture_output=True, text=True, check=True, + ) + result = subprocess.run( + ["terraform", "plan", "-input=false", "-lock=false", "-refresh=false", + "-no-color", "-var-file=inputs.tfvars.json", "-out=plan"], + cwd=fixture, capture_output=True, text=True, + ) + if result.returncode: + return result, None + plan = subprocess.run( + ["terraform", "show", "-json", "plan"], + cwd=fixture, capture_output=True, text=True, check=True, + ) + rendered = json.loads(plan.stdout)["planned_values"]["outputs"]["rendered"]["value"] + return result, sandboxd_config(rendered) + + def test_cloud_defaults_and_explicit_memory_policy_overrides(self): + for provider in ("aliyun", "huaweicloud"): + for inputs, enabled, reserve in ( + ({}, False, "16GiB"), + ({"sandbox_memory_limit_enabled": True}, True, "16GiB"), + ({"sandbox_memory_limit_enabled": True, "sandbox_memory_total_reserve": "2GiB"}, True, "2GiB"), + ({"sandbox_memory_limit_enabled": False, "sandbox_memory_total_reserve": "2GiB"}, False, "2GiB"), + ): + with self.subTest(provider=provider, inputs=inputs): + result, config = self.render(provider, inputs) + self.assertEqual(result.returncode, 0, result.stderr) + resource = config["plugin"]["resource"] + self.assertEqual(resource["memory_limit"], {"enabled": enabled, "total_reserve": reserve}) + self.assertIs(resource["memory_limit"]["enabled"], enabled) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual(config["plugin"]["image"]["cgroup_memory_limit"], "64GiB") + + def test_invalid_reserves_fail_before_deployment(self): + for provider in ("aliyun", "huaweicloud"): + for reserve in ("", "0", "0GiB", "16G", "auto", "-1GiB", "16GiB\"\nenabled=true"): + with self.subTest(provider=provider, reserve=reserve): + result, _ = self.render(provider, {"sandbox_memory_total_reserve": reserve}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("sandbox_memory_total_reserve", result.stderr) + + +class GuidedMemoryPolicyTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.fixture = Path(self.temporary.name) + scripts = self.fixture / "deploy/scripts" + scripts.mkdir(parents=True) + for name in ("configure.sh", "common.sh", "shared_storage_config.py"): + shutil.copy2(ROOT.parent / "scripts" / name, scripts) + shutil.copy2(ROOT.parents[1] / "Makefile", self.fixture) + policy = self.fixture / "deploy/terraform/aliyun/node-image-pull-policy.json" + policy.parent.mkdir(parents=True) + shutil.copy2(ROOT / "aliyun" / policy.name, policy) + commands = self.fixture / "bin" + commands.mkdir() + formatter = commands / "terraform" + # Profile generation only calls terraform fmt; evaluated rendering and + # validation are covered separately with the real Terraform CLI above. + formatter.write_text('#!/bin/sh\n[ "$1" = fmt ]\n') + formatter.chmod(0o755) + self.environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ["PATH"]) + for name in ("SANDBOX_MEMORY_LIMIT_ENABLED", "SANDBOX_MEMORY_TOTAL_RESERVE", "FORCE"): + self.environment.pop(name, None) + + def generate(self, provider, name, *settings): + return subprocess.run( + ["make", "config", f"VENDOR={provider}", f"ENV={name}", "NON_INTERACTIVE=1", + "NODE_POOL_KEY_NAME=fixture", "IMAGE_REPOSITORY=example.invalid/fixture", + "IMAGE_TAG=fixture", *settings], + cwd=self.fixture, env=self.environment, capture_output=True, text=True, + ) + + def test_generated_profiles_default_to_opt_out_and_preserve_explicit_opt_in(self): + for provider in ("aliyun", "huaweicloud"): + for name, settings, enabled, reserve in ( + ("default", (), "false", "16GiB"), + ("enabled", ("SANDBOX_MEMORY_LIMIT_ENABLED=true",), "true", "16GiB"), + ("small", ("SANDBOX_MEMORY_LIMIT_ENABLED=true", "SANDBOX_MEMORY_TOTAL_RESERVE=2GiB"), "true", "2GiB"), + ): + profile_name = f"{provider}-{name}" + with self.subTest(provider=provider, profile=name): + result = self.generate(provider, profile_name, *settings) + self.assertEqual(result.returncode, 0, result.stderr) + profile = self.fixture / ".akernel" / profile_name + tfvars = (profile / "terraform.tfvars").read_text() + config = (profile / "config.env").read_text() + self.assertIn(f"sandbox_memory_limit_enabled = {enabled}\n", tfvars) + self.assertIn(f'sandbox_memory_total_reserve = "{reserve}"\n', tfvars) + self.assertIn(f"SANDBOX_MEMORY_LIMIT_ENABLED={enabled}\n", config) + self.assertIn(f"SANDBOX_MEMORY_TOTAL_RESERVE={reserve}\n", config) + + def test_invalid_input_and_existing_profiles_are_not_replaced(self): + for provider in ("aliyun", "huaweicloud"): + with self.subTest(provider=provider): + result = self.generate(provider, provider) + self.assertEqual(result.returncode, 0, result.stderr) + profile = self.fixture / ".akernel" / provider + original = {name: (profile / name).read_bytes() for name in ("config.env", "terraform.tfvars")} + for settings in ( + ("SANDBOX_MEMORY_LIMIT_ENABLED=maybe",), + ("SANDBOX_MEMORY_TOTAL_RESERVE=0GiB",), + ("SANDBOX_MEMORY_TOTAL_RESERVE=16G",), + ("SANDBOX_MEMORY_LIMIT_ENABLED=true",), + ): + result = self.generate(provider, provider, *settings) + self.assertNotEqual(result.returncode, 0) + for name, data in original.items(): + self.assertEqual((profile / name).read_bytes(), data) def evaluate_fixture(main, inputs, files=None): @@ -152,6 +318,33 @@ def test_managed_clients_cannot_target_non_linux_nodes(self): class AdxTerraformContractTest(unittest.TestCase): + def test_cloud_memory_policy_is_explicitly_wired_and_defaults_off(self): + for provider in ("aliyun", "huaweicloud"): + with self.subTest(provider=provider): + values = (ROOT / provider / "values-akernel.yaml.tmpl").read_text() + main = (ROOT / provider / "main.tf").read_text() + variables = (ROOT / provider / "variables.tf").read_text() + for name, default in zip(MEMORY_INPUTS, ("false", '"16GiB"')): + self.assertRegex(main, rf"{name}\s*= var\.{name}") + variable = re.search(rf'(?ms)^variable "{name}" {{.*?^}}', variables).group() + self.assertRegex(variable, rf"default\s*= {default}") + self.assertIn("enabled = ${sandbox_memory_limit_enabled}", values) + self.assertIn("total_reserve = ${jsonencode(sandbox_memory_total_reserve)}", values) + config = sandboxd_config(values.replace("${sandbox_memory_limit_enabled}", "false").replace( + "${jsonencode(sandbox_memory_total_reserve)}", '"16GiB"')) + resource = config["plugin"]["resource"] + self.assertEqual( + resource["memory_limit"], + {"enabled": False, "total_reserve": "16GiB"}, + ) + self.assertIs(resource["memory_limit"]["enabled"], False) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual( + config["plugin"]["image"]["cgroup_memory_limit"], "64GiB" + ) + def test_each_provider_exposes_adx_control_and_data_directly(self) -> None: for provider in ("aliyun", "huaweicloud"): with self.subTest(provider=provider):