From ffc935ac538397166c9bdb5e118111218c1952db Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Thu, 5 Nov 2026 18:00:45 +0800 Subject: [PATCH 1/9] feat(deploy): enable parent memory budgeting by default Enable the aggregate sandbox memory policy with a 16 GiB total reserve in the standalone, Helm, Aliyun and Huawei templates. Document the cgroup v2 and effective-memory requirements, explicit overrides and migration of existing sandbox subtrees. Keep the policy enabled in small CI and PVM fixtures with an explicit 2 GiB reserve, and verify that the parent hard limit matches resource reporting. Add configuration-generation and override regressions across deployment paths. The default is deployment policy, not a host OOM safety guarantee. Signed-off-by: wanghongtao.wangho --- .github/workflows/ci.yml | 39 ++++++++++++++ AGENTS.md | 10 ++++ deploy/README.md | 51 +++++++++++++++++++ .../tests/test_shared_checkpoint_storage.py | 48 ++++++++++++++++- deploy/akernel/charts/core/values.yaml | 6 +-- deploy/pvm-runtime.md | 7 +++ deploy/pvm/run-adx-ci.sh | 4 ++ deploy/standalone/README.md | 11 ++++ deploy/standalone/config/sandboxd_config.toml | 6 +-- deploy/standalone/test_start_config.py | 34 +++++++++++++ .../terraform/aliyun/values-akernel.yaml.tmpl | 6 +-- .../huaweicloud/values-akernel.yaml.tmpl | 6 +-- deploy/terraform/test_adx_contract.py | 24 +++++++++ 13 files changed, 239 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab4fc37b..a4bfc56c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,6 +257,23 @@ jobs: test ! -e /usr/local/bin/runc-shim ' + - name: Select the bounded CI memory profile + run: | + # The runner can have less memory than the deployment's 16 GiB reserve. + # Exercise the enabled policy with an explicit smaller test reserve. + python3 - <<'PY' + from pathlib import Path + import tomllib + path = Path("deploy/standalone/config/sandboxd_config.toml") + text = path.read_text() + assert tomllib.loads(text)["plugin"]["resource"]["memory_limit"] == { + "enabled": True, "total_reserve": "16GiB" + } + old = 'total_reserve = "16GiB"' + assert text.count(old) == 1 + path.write_text(text.replace(old, 'total_reserve = "2GiB"', 1)) + PY + - name: Start standalone AKernel run: | IMAGE="akernel-ci/all-in-one:${GITHUB_SHA}" \ @@ -265,6 +282,28 @@ jobs: AKERNEL_DATA_BIND=127.0.0.1 \ ./deploy/standalone/start.sh + - name: Verify the enabled parent memory policy + run: | + docker exec -i akernel-node python3 - <<'PY' + import json + import socket + from http.client import HTTPConnection + from pathlib import Path + maximum = int(Path("/sys/fs/cgroup/akernel/memory.max").read_text()) + assert maximum > 0 + connection = HTTPConnection("localhost", timeout=10) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(10) + connection.sock.connect("/var/run/resource.sock") + connection.request("GET", "/resource") + response = connection.getresponse() + assert response.status == 200, response.status + capacity = json.load(response)["mem"] + connection.close() + assert capacity == maximum, (capacity, maximum) + print(f"finite parent memory.max and /resource agree: {maximum} bytes") + PY + - name: Trust the standalone HTTPS certificate run: | docker cp akernel-node:/home/akernel/adx/tls/ingress-public.pem \ diff --git a/AGENTS.md b/AGENTS.md index 0ff04e8b..956dea07 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,6 +240,16 @@ distill-fs release's packaged manifest. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and Terraform-based cloud provisioning. +AKernel's maintained node templates enable the cgroup v2 parent memory policy +with `total_reserve="16GiB"`. Effective memory (host MemTotal clipped by visible +ancestor limits) must exceed that reserve and leave room for sandbox requests. +Small Docker/OrbStack VMs and CI fixtures must explicitly select a smaller +reserve or disable the policy in their sandboxd configuration. Keep standalone, +Helm and both cloud templates aligned. Adding or lowering a parent cap requires +an empty sandbox subtree; an existing stricter cap is retained. This aggregate +sandbox cap does not bound all node components or guarantee avoidance of host OOM. +See `deploy/README.md#aggregate-sandbox-memory-budget` for migration and scope. + The [standalone guide](./deploy/standalone/README.md) owns source startup, supported platforms, runtime selection, and SDK setup. Preserve the source launcher's native-daemon architecture selection, clean sandboxd gitlink check, occupied-name preflight, profile locking, unique source image, and node/gateway health checks. Status and stop must verify recorded container identities, images, and data mounts; stop also requires an empty sandbox inventory. Source management is independent of cloud profiles and must preserve unrelated containers. The all-in-one image and node launchers declare lowercase `container=oci` diff --git a/deploy/README.md b/deploy/README.md index ca066b1d..bb054f2b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,6 +17,57 @@ when a shared filesystem is independently required. For one deployment serving both native and E2B SDKs with version-independent application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-application-logs.md). +## Aggregate sandbox memory budget + +The standalone, Helm and both cloud templates enable the cgroup v2 aggregate +sandbox memory policy by default: + +```toml +[plugin.resource.memory_limit] +enabled = true +total_reserve = "16GiB" +``` + +Effective node memory is the minimum of host `MemTotal` and visible finite +ancestor cgroup limits. Sandboxd subtracts the reserve, rounds the result down +to a page boundary and preserves any existing stricter `/akernel/memory.max`. +Adxlet's resource source is clipped to that effective parent ceiling, without +subtracting the reserve a second time. For example, effective memory of 64 GiB +and this reserve produce a candidate of 48 GiB; an existing 40 GiB cap remains +40 GiB. + +The default requires cgroup v2 and effective memory greater than 16 GiB, with +enough remaining capacity for the requested sandboxes. Docker/OrbStack VM limits +and visible ancestor limits participate in this calculation. Use a matching node +image built with the pinned sandboxd's `total_reserve` support. On a smaller node, +explicitly choose a smaller positive `total_reserve` or set `enabled=false` in +the sandboxd TOML before starting. Standalone reads its source config template; +Helm uses `node.config.sandboxd.config` (`core.node.config.sandboxd.config` in the +umbrella chart). Cloud templates provide the complete TOML and must be adjusted +there or through the existing complete-config override. Values use integer +bytes or `B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not +accepted. Standard and PVM CI fixtures explicitly use a 2 GiB reserve while +keeping the policy enabled. + +This reserve is deployment policy, not a measured component budget or a +guarantee against host OOM. Distill-fs retains its independent 64 GiB limit and +is not automatically deducted. Control components, other Pods and host kernel +memory remain outside the sandbox parent; their limits, reclaim behavior and +peak demand need separate consideration. In particular, kernel memory excluded +by a host's `cgroup.memory=nokmem` setting is not covered by the parent limit. + +For existing nodes, adding a finite cap to an unlimited parent or lowering a +cap requires an empty sandbox subtree and residual usage below the new limit. +An equal or stricter existing cap can be reused with live sandboxes; a larger +candidate never automatically raises it. Review the rollout before upgrading: +ConfigMap checksums trigger Pod replacement, and the current chart permits +200 unavailable nodes. Use a canary or a separately constrained rollout after +draining, rather than enabling the policy across live nodes together. Custom +TOML containing the removed reserve fields must be migrated even when the +policy is disabled. See the pinned sandboxd +[memory-limit guide](../src/sandboxd/doc/node-memory-limit.md) for migration and +the complete startup contract. + ## Guided deployment The image installs the complete gVisor bundle pinned by sandboxd's runtime manifest: runsc, the containerd shim, and adjacent `gvisor-bin/` helpers. Its SHA-512 verifies the archive, not a standalone runsc binary. Keep these executables together when building custom images; replacing only runsc can leave mismatched checkpoint/restore helpers. diff --git a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py index 01adb83e..4dffb59d 100644 --- a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py +++ b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py @@ -5,6 +5,7 @@ from __future__ import annotations import subprocess +import tempfile import tomllib import unittest from pathlib import Path @@ -14,10 +15,14 @@ CHART = Path(__file__).resolve().parents[1] -def render(*settings: str) -> dict[tuple[str, str], dict]: +def render( + *settings: str, set_files: dict[str, Path] | None = None +) -> dict[tuple[str, str], dict]: command = ["helm", "template", "akernel", str(CHART), "--namespace", "akernel-test"] for setting in settings: command.extend(["--set-string", setting]) + for key, path in (set_files or {}).items(): + command.extend(["--set-file", f"{key}={path}"]) result = subprocess.run(command, capture_output=True, text=True, check=True) return { (item["kind"], item["metadata"]["name"]): item @@ -27,6 +32,47 @@ def render(*settings: str) -> dict[tuple[str, str], dict]: class SharedCheckpointStorageTests(unittest.TestCase): + def assert_memory_config(self, config, enabled, reserve): + resource = config["plugin"]["resource"] + self.assertEqual( + resource["memory_limit"], + {"enabled": enabled, "total_reserve": reserve}, + ) + self.assertIs(resource["memory_limit"]["enabled"], enabled) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual(config["plugin"]["image"]["cgroup_memory_limit"], "64GiB") + + def test_default_parent_memory_limit_uses_16gib_total_reserve(self): + config = tomllib.loads( + render()[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + ) + self.assert_memory_config(config, True, "16GiB") + + def test_complete_config_override_preserves_disabled_and_small_node_policy(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for enabled, reserve in ((False, "16GiB"), (True, "2GiB")): + with self.subTest(enabled=enabled, reserve=reserve): + custom = default.replace( + "enabled = true", f"enabled = {str(enabled).lower()}", 1 + ).replace( + 'total_reserve = "16GiB"', f'total_reserve = "{reserve}"', 1 + ) + path.write_text(custom) + resources = render(set_files={"node.config.sandboxd.config": path}) + rendered = resources[("ConfigMap", "akernel-node-config")]["data"][ + "sandboxd_config.toml" + ] + self.assertEqual(rendered.rstrip(), custom.rstrip()) + self.assert_memory_config(tomllib.loads(rendered), enabled, reserve) + def test_control_service_budgets_protect_voluntary_disruption(self): for replicas in (1, 3): resources = render(f"adx.ingressApi.replicas={replicas}") diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 3b1b2b9f..8972851c 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -532,9 +532,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + enabled = true + # Requires effective memory above this reserve; tune for the deployment. + total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index fb17510c..fd9c6859 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -193,6 +193,13 @@ The dedicated PVM qualification omits runsc and Kata runtime entries from its te The PVM image applies `prepare-adx-config.py --pvm` to the shared standalone profile, setting coordinator and adxlet `rpc_timeout_seconds` to 90. Custom/externally supplied ADX configurations must carry these overrides themselves. The ordinary KVM image keeps its original ADX profile. This is a bounded runtime cleanup/reload budget: a recorded cleanup took 30.63 seconds and exceeded the old 30-second limit. The guest-agent startup budget stays at 15 seconds. +The maintained PVM CI fixture selects a 2 GiB aggregate sandbox memory reserve +for its 8 GiB L1, keeping the memory policy enabled. This explicit qualification +profile is smaller than the 16 GiB deployment reserve; it does not qualify that +deployment default or establish a host OOM safety budget. +Earlier qualification runs predate this enabled finite-cap fixture; the updated +profile requires rerunning PVM qualification. + ## Real Actions qualification The [independent sandboxd PVM job](https://github.com/hky1999/sandboxd/actions/runs/38558942312), source `54d7fd878abe93db48b755f9063f7fdf3bc3d728`, passed Full, Incremental and virtio-fs suites with ten consecutive checkpoints each. It covered checkpoint-after-restore chains, OCI image-provider paths, network ACLs/DNAT, quota and EROFS/directory writable host mounts, rotation and read-offset preservation. All three exits were zero; host-kernel WARN/Oops checks and module cleanup passed. The external Redis DNAT client now waits for the suite's explicit Redis-ready stage before starting the existing DNAT readiness budget; earlier checkpoint work no longer consumes that budget. diff --git a/deploy/pvm/run-adx-ci.sh b/deploy/pvm/run-adx-ci.sh index 80e03220..7fd12e4c 100644 --- a/deploy/pvm/run-adx-ci.sh +++ b/deploy/pvm/run-adx-ci.sh @@ -119,6 +119,10 @@ assert text.count(old)==1 text=text.replace(old,'resolv_conf_path="/home/akernel/sandboxd/config/pvm-ci-resolv.conf"') text=text.replace('filestore_dir_size="50G"','filestore_dir_size=""') text=text.replace('filestore_xfs_enabled=false','filestore_xfs_enabled=true') +# The dedicated qualification L1 has 8 GiB; keep the parent policy enabled +# with an explicit test reserve instead of the 16 GiB deployment default. +assert text.count('total_reserve = "16GiB"')==1 +text=text.replace('total_reserve = "16GiB"','total_reserve = "2GiB"',1) # OCI and Nydus directory roots require an explicitly selected virtio-fs # profile; the maintained standalone default remains EROFS. assert text.count('virtiofs_enabled=false')==1 diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index d17d4ba5..3e9159f4 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -58,6 +58,17 @@ for inspection and retry. Data and credentials survive stopping. For an independent data profile, set an absolute `AKERNEL_STANDALONE_DATA_DIR` for every command. Only one standalone node can run per Docker daemon. +Standalone enables the aggregate sandbox memory limit with a 16 GiB reserve. +It requires cgroup v2 and effective memory above that reserve, including any +Docker/OrbStack VM or visible ancestor limit. Before starting on a smaller +machine, edit `[plugin.resource.memory_limit]` in +`deploy/standalone/config/sandboxd_config.toml` to select a smaller positive +`total_reserve` or explicitly disable the policy. Editing only the generated +data-directory TOML is insufficient because startup regenerates it. This also +applies to `make standalone-source-test`. See +[the memory budget and upgrade contract](../README.md#aggregate-sandbox-memory-budget) +before applying the new default to an existing profile. + ### Runtime selection Runsc is the source default. Opt into runc with diff --git a/deploy/standalone/config/sandboxd_config.toml b/deploy/standalone/config/sandboxd_config.toml index 240161e3..354f88f8 100644 --- a/deploy/standalone/config/sandboxd_config.toml +++ b/deploy/standalone/config/sandboxd_config.toml @@ -29,9 +29,9 @@ max_instance_num=1000 enable_cache_in_stock_mode=true [plugin.resource.memory_limit] -enabled = false -# Set a deployment-specific total reserve before enabling. -# total_reserve = "22GiB" +enabled = true +# Requires effective memory above this reserve; tune for the deployment. +total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index b936fb1d..31c38904 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -93,6 +93,40 @@ def assert_rejected_without_replacing_config(self, message): self.assertEqual(self.config_output.read_text(), "existing config\n") self.assertFalse(self.resolver_output.exists()) + def assert_memory_config(self, enabled, reserve): + config = self.read_sections() + self.assertEqual( + dict(config["plugin.resource.memory_limit"]), + {"enabled": enabled, "total_reserve": f'"{reserve}"'}, + ) + self.assertEqual(config["plugin.resource"]["cgroup_root_name"], '"/akernel"') + self.assertEqual(config["plugin.resource"]["max_instance_num"], "1000") + self.assertEqual( + config["plugin.resource"]["enable_cache_in_stock_mode"], "true" + ) + self.assertEqual(config["plugin.image"]["cgroup_memory_limit"], '"64GiB"') + + def test_generated_config_enables_parent_limit_with_16gib_reserve(self): + result = self.generate(AKERNEL_ENABLE_RUNC="false") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config("true", "16GiB") + + def test_generated_config_preserves_explicit_memory_policy_overrides(self): + original = self.template.read_text() + for enabled, reserve in (("false", "16GiB"), ("true", "2GiB")): + with self.subTest(enabled=enabled, reserve=reserve): + custom, count = re.subn( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "[plugin.resource.memory_limit]\n" + f"enabled = {enabled}\ntotal_reserve = \"{reserve}\"\n\n", + original, + ) + self.assertEqual(count, 1) + self.template.write_text(custom) + result = self.generate(AKERNEL_ENABLE_RUNC="false") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, reserve) + def test_pvm_selection_preserves_direct_resolver_configuration(self): result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") self.assertEqual(result.returncode, 0, result.stderr) diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 0a56d1c0..5b3c2e25 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -165,9 +165,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + enabled = true + # Requires effective memory above this reserve; tune for the deployment. + total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 031b4209..ed47ff35 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -142,9 +142,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = false - # Set a deployment-specific total reserve before enabling. - # total_reserve = "22GiB" + enabled = true + # Requires effective memory above this reserve; tune for the deployment. + total_reserve = "16GiB" [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/test_adx_contract.py b/deploy/terraform/test_adx_contract.py index 0fc61196..5feae04e 100755 --- a/deploy/terraform/test_adx_contract.py +++ b/deploy/terraform/test_adx_contract.py @@ -9,6 +9,8 @@ import shutil import subprocess import tempfile +import textwrap +import tomllib import unittest from pathlib import Path @@ -176,6 +178,28 @@ def test_managed_clients_cannot_target_non_linux_nodes(self): class AdxTerraformContractTest(unittest.TestCase): + def test_each_provider_enables_parent_limit_with_16gib_total_reserve(self): + for provider in ("aliyun", "huaweicloud"): + with self.subTest(provider=provider): + values = (ROOT / provider / "values-akernel.yaml.tmpl").read_text() + block = re.search( + r"(?ms)^ config: \|-\n(.*?)(?=^ {0,6}\S|\Z)", values + ) + self.assertIsNotNone(block) + config = tomllib.loads(textwrap.dedent(block.group(1))) + resource = config["plugin"]["resource"] + self.assertEqual( + resource["memory_limit"], + {"enabled": True, "total_reserve": "16GiB"}, + ) + self.assertIs(resource["memory_limit"]["enabled"], True) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual( + config["plugin"]["image"]["cgroup_memory_limit"], "64GiB" + ) + def test_each_provider_exposes_adx_control_and_data_directly(self) -> None: for provider in ("aliyun", "huaweicloud"): with self.subTest(provider=provider): From 49fdde41ad8654d89653f8674e994ea2a18c3787 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Thu, 5 Nov 2026 18:15:42 +0800 Subject: [PATCH 2/9] docs(deploy): record default rollout prerequisites Document the observed standalone ADX capacity-source mismatch and the cloud operating-system requirements before treating the new memory policy as a deployment-wide default. Signed-off-by: wanghongtao.wangho --- deploy/README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index bb054f2b..612ef0cf 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -31,10 +31,16 @@ total_reserve = "16GiB" Effective node memory is the minimum of host `MemTotal` and visible finite ancestor cgroup limits. Sandboxd subtracts the reserve, rounds the result down to a page boundary and preserves any existing stricter `/akernel/memory.max`. -Adxlet's resource source is clipped to that effective parent ceiling, without -subtracting the reserve a second time. For example, effective memory of 64 GiB -and this reserve produce a candidate of 48 GiB; an existing 40 GiB cap remains -40 GiB. +The sandboxd resource socket is clipped to that effective parent ceiling, +without subtracting the reserve a second time. Helm and cloud Adxlet configs +consume that socket. For example, effective memory of 64 GiB and this reserve +produce a candidate of 48 GiB; an existing 40 GiB cap remains 40 GiB. + +Standalone currently inherits ADX's `auto` capacity source, which observes the +Adxlet's own cgroup rather than the sandbox parent. Its public scheduling +capacity can therefore exceed the parent cap even when `/resource` is correct. +The standalone capacity source must be wired to sandboxd and verified before +relying on the public scheduling capacity to match the new limit. The default requires cgroup v2 and effective memory greater than 16 GiB, with enough remaining capacity for the requested sandboxes. Docker/OrbStack VM limits @@ -49,6 +55,11 @@ bytes or `B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not accepted. Standard and PVM CI fixtures explicitly use a 2 GiB reserve while keeping the policy enabled. +Verify the operating-system choice for cloud profiles as well. The current +Aliyun `node_pool_image_type="AliyunLinux3"` default is the standard image, +which uses cgroup v1; select and qualify a cgroup v2 image before enabling this +policy. Huawei's selected node image also requires cgroup-mode verification. + This reserve is deployment policy, not a measured component budget or a guarantee against host OOM. Distill-fs retains its independent 64 GiB limit and is not automatically deducted. Control components, other Pods and host kernel From abcc427a8851f57a1c2401d3de9b18cc94cca1a0 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 10:44:36 +0800 Subject: [PATCH 3/9] fix(deploy): align sandbox capacity and gate cloud memory defaults Resolve the bundled standalone ADX profile at image build time before replacing its complete capacity source with sandboxd. Preserve explicit custom sources and verify resolved standard, PVM, trace and shared-storage configurations. Keep unqualified cloud OS profiles opt-in, exposing enabled and reserve inputs through Terraform and guided configuration. Add public scheduling capacity and allocation-release E2E coverage alongside the parent/socket assertion. Signed-off-by: wanghongtao.wangho --- .github/workflows/ci.yml | 1 + AGENTS.md | 24 ++- Makefile | 5 + builder/node.Dockerfile | 10 +- builder/scripts/prepare-adx-config.py | 54 +++++- builder/scripts/test_afs_config.py | 121 ++++++++++++ deploy/README.md | 36 ++-- deploy/pvm-runtime.md | 8 +- deploy/scripts/configure.sh | 26 +++ deploy/standalone/README.md | 8 + deploy/terraform/aliyun/README.md | 21 ++ deploy/terraform/aliyun/main.tf | 3 + .../terraform/aliyun/terraform.tfvars.example | 5 + .../terraform/aliyun/values-akernel.yaml.tmpl | 6 +- deploy/terraform/aliyun/variables.tf | 16 ++ deploy/terraform/huaweicloud/README.md | 21 ++ deploy/terraform/huaweicloud/main.tf | 3 + .../huaweicloud/terraform.tfvars.example | 5 + .../huaweicloud/values-akernel.yaml.tmpl | 6 +- deploy/terraform/huaweicloud/variables.tf | 16 ++ deploy/terraform/test_adx_contract.py | 180 +++++++++++++++++- .../e2e/standalone/test_sdk_contracts.py | 50 +++++ 22 files changed, 586 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4bfc56c..94eea1c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,6 +303,7 @@ jobs: assert capacity == maximum, (capacity, maximum) print(f"finite parent memory.max and /resource agree: {maximum} bytes") PY + echo "AKERNEL_TEST_EXPECT_MEMORY_BYTES=$(docker exec akernel-node cat /sys/fs/cgroup/akernel/memory.max)" >> "${GITHUB_ENV}" - name: Trust the standalone HTTPS certificate run: | diff --git a/AGENTS.md b/AGENTS.md index 956dea07..f5cc9bd9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,12 +240,21 @@ distill-fs release's packaged manifest. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and Terraform-based cloud provisioning. -AKernel's maintained node templates enable the cgroup v2 parent memory policy -with `total_reserve="16GiB"`. Effective memory (host MemTotal clipped by visible -ancestor limits) must exceed that reserve and leave room for sandbox requests. +Standalone and the direct Helm chart enable the cgroup v2 parent memory policy +with `total_reserve="16GiB"`. Both cloud Terraform profiles leave it disabled +until explicitly enabled with `sandbox_memory_limit_enabled`; their default +operating-system images have not been qualified for this policy. Guided profiles +expose `SANDBOX_MEMORY_LIMIT_ENABLED` and `SANDBOX_MEMORY_TOTAL_RESERVE`. +Effective memory (host MemTotal clipped by visible ancestor limits) must exceed +that reserve and leave room for sandbox requests. Small Docker/OrbStack VMs and CI fixtures must explicitly select a smaller -reserve or disable the policy in their sandboxd configuration. Keep standalone, -Helm and both cloud templates aligned. Adding or lowering a parent cap requires +reserve or disable the policy in their sandboxd configuration. Keep the budget +semantics aligned across standalone, Helm and both cloud templates. +The bundled standalone ADX configuration is resolved at image build time and +uses sandboxd's resource socket; replace the complete tagged capacity source, +preserving explicitly selected custom sources. Public scheduling capacity must +follow the sandbox parent ceiling, with allocatable decreasing and recovering +as requests are allocated and deleted. Adding or lowering a parent cap requires an empty sandbox subtree; an existing stricter cap is retained. This aggregate sandbox cap does not bound all node components or guarantee avoidance of host OOM. See `deploy/README.md#aggregate-sandbox-memory-budget` for migration and scope. @@ -725,6 +734,11 @@ entirely. Do not publish per-commit SHA tags or other historical image tags. The job checks the image contains runsc and excludes Kata, Firecracker, virtiofsd, and runc before starting standalone E2E. +The bounded standalone CI profile selects a 2 GiB reserve and exports the +measured parent limit as `AKERNEL_TEST_EXPECT_MEMORY_BYTES`. The SDK E2E +regression checks public capacity plus allocation and release against that +single-node budget; retain the parent/socket check as a separate assertion. + CI explicitly trusts the generated standalone public certificate through `SSL_CERT_FILE` and enables `AKERNEL_VERIFY_TLS=1` before SDK examples and integration tests. Keep HTTPS certificate and hostname validation enabled. diff --git a/Makefile b/Makefile index 2d14edfb..5bc4bb24 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,8 @@ MONITOR_STORAGE_CLASS ?= INSTALL_MONITOR ?= INSTALL_DRAGONFLY ?= CHUNK_DB_SIZE ?= +SANDBOX_MEMORY_LIMIT_ENABLED ?= +SANDBOX_MEMORY_TOTAL_RESERVE ?= JUICEFS_ENABLED ?= false JUICEFS_STORAGE_CLASS ?= JUICEFS_CACHE_MIB ?= 4096 @@ -50,6 +52,7 @@ help: @echo " make config NON_INTERACTIVE=1 ... Generate config from Make variables" @echo " make config INSTALL_DRAGONFLY=true Enable optional P2P image distribution" @echo " make config ENABLE_RUNC=true Build and register the optional runc runtime" + @echo " make config SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB Opt into a qualified cgroup v2 sandbox budget" @echo " make config JUICEFS_ENABLED=true JUICEFS_STORAGE_CLASS=... Deploy JuiceFS/PostgreSQL/Ceph" @echo " make config CHECKPOINT_STORAGE_HOST_PATH=/srv/shared Use an existing AFS/JuiceFS mount" @echo " make config SCHEDULE_PLACEMENT_POLICY=binpack Use compact scheduling" @@ -101,6 +104,8 @@ config: if [[ -n "$(INSTALL_MONITOR)" ]]; then args+=(--install-monitor "$(INSTALL_MONITOR)"); fi; \ if [[ -n "$(INSTALL_DRAGONFLY)" ]]; then args+=(--install-dragonfly "$(INSTALL_DRAGONFLY)"); fi; \ if [[ -n "$(CHUNK_DB_SIZE)" ]]; then args+=(--chunk-db-size "$(CHUNK_DB_SIZE)"); fi; \ + if [[ -n "$(SANDBOX_MEMORY_LIMIT_ENABLED)" ]]; then args+=(--sandbox-memory-limit-enabled "$(SANDBOX_MEMORY_LIMIT_ENABLED)"); fi; \ + if [[ -n "$(SANDBOX_MEMORY_TOTAL_RESERVE)" ]]; then args+=(--sandbox-memory-total-reserve "$(SANDBOX_MEMORY_TOTAL_RESERVE)"); fi; \ args+=(--juicefs-enabled "$(JUICEFS_ENABLED)"); \ args+=(--juicefs-cache-mib "$(JUICEFS_CACHE_MIB)" --juicefs-cache-large-write "$(JUICEFS_CACHE_LARGE_WRITE)"); \ if [[ -n "$(JUICEFS_CACHE_HOST_PATH)" ]]; then args+=(--juicefs-cache-host-path "$(JUICEFS_CACHE_HOST_PATH)"); fi; \ diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 708a7e98..51f03fdd 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -476,11 +476,13 @@ COPY ./builder/scripts/sandboxd_network_prepare.sh /usr/local/bin/sandboxd-netwo ARG FIRECRACKER_KERNEL_PROFILE COPY ./builder/config/adx-standalone.yaml /etc/akernel/adx-standalone.yaml RUN case "${FIRECRACKER_KERNEL_PROFILE}" in \ - pvm) python3 /usr/local/libexec/prepare-adx-config.py \ - /etc/akernel/adx-standalone.yaml /etc/akernel/adx-standalone.yaml --pvm ;; \ - kvm) ;; \ + pvm) set -- --pvm ;; \ + kvm) set -- ;; \ *) echo "invalid Firecracker kernel profile" >&2; exit 1 ;; \ - esac + esac; \ + python3 /usr/local/libexec/prepare-adx-config.py \ + /etc/akernel/adx-standalone.yaml /etc/akernel/adx-standalone.yaml \ + --sandboxd-capacity "$@" RUN chmod 0755 \ /usr/local/bin/runsc \ /usr/local/bin/sandboxd \ diff --git a/builder/scripts/prepare-adx-config.py b/builder/scripts/prepare-adx-config.py index 26901917..c114920e 100644 --- a/builder/scripts/prepare-adx-config.py +++ b/builder/scripts/prepare-adx-config.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Apply PVM, telemetry and mounted checkpoint options to ADX deployments.""" +"""Prepare the bundled capacity source, PVM, telemetry and checkpoint options.""" from __future__ import annotations import argparse import os @@ -10,6 +10,50 @@ import yaml +def standalone_sandboxd_capacity(cfg: dict, source: Path, adxctl: Path) -> dict: + """Resolve the static bundled profile before replacing its tagged source. + + ADX recursively merges profile overrides, which would retain auto's + disk_path when switching to sandboxd. Custom configurations with an + explicit source remain caller-owned, including explicit auto/file sources. + This build-time conversion does not expand custom runtime expressions. + """ + if cfg.get("profile") != "standalone": + return cfg + node = cfg.get("service_overrides", {}).get("adxlet", {}).get("config") or {} + if "capacity_file" in node or node.get("resource_source") != { + "disk_path": "/home/akernel" + }: + return cfg + + def has_expression(value): + if isinstance(value, str): + return "$" in value + if isinstance(value, dict): + return any(has_expression(item) for item in value.values()) + if isinstance(value, list): + return any(has_expression(item) for item in value) + return False + + if has_expression(cfg): + raise ValueError("bundled capacity conversion requires a static deployment") + resolved = yaml.safe_load(subprocess.run( + [str(adxctl), "--config", str(source), "config", "dump"], + check=True, capture_output=True, text=True, + ).stdout) + if not isinstance(resolved, dict) or "profile" in resolved: + raise ValueError("ADX did not resolve the standalone deployment") + workers = [service for service in resolved["services"] + if service["role"] == "adxlet"] + if len(workers) != 1: + raise ValueError("bundled standalone deployment must contain one adxlet") + workers[0]["config"]["resource_source"] = { + "kind": "sandboxd", "socket": "/var/run/resource.sock", + "valid_for_seconds": 30, + } + return resolved + + def expand_selector(value: str) -> str: """Match ADX's single-pass ${NAME:-default}/$$ discriminator expansion. @@ -52,10 +96,13 @@ def expand_selector(value: str) -> str: def configure( source: Path, destination: Path, mount: Path | None, prefix: str, *, pvm: bool = False, + sandboxd_capacity: bool = False, adxctl: Path = Path("/opt/adx/current/bin/adxctl"), ) -> None: cfg = yaml.safe_load(source.read_text()) if not isinstance(cfg, dict): raise ValueError("deployment must be a mapping") + if sandboxd_capacity: + cfg = standalone_sandboxd_capacity(cfg, source, adxctl) profile_roles = { "standalone": ("coordinator", "apiserver", "adxlet", "ingress"), "standalone-external-redis": ("coordinator", "apiserver", "adxlet", "ingress"), @@ -134,8 +181,11 @@ def main() -> None: parser.add_argument("--mount", type=Path, default=None) parser.add_argument("--prefix", default="akernel/checkpoints") parser.add_argument("--pvm", action="store_true", help="use the PVM cleanup RPC budget") + parser.add_argument("--sandboxd-capacity", action="store_true", + help="resolve the static bundled profile and use sandboxd capacity") args = parser.parse_args() - configure(args.source, args.destination, args.mount, args.prefix, pvm=args.pvm) + configure(args.source, args.destination, args.mount, args.prefix, pvm=args.pvm, + sandboxd_capacity=args.sandboxd_capacity) if __name__ == "__main__": diff --git a/builder/scripts/test_afs_config.py b/builder/scripts/test_afs_config.py index cb32999b..4d19bb37 100644 --- a/builder/scripts/test_afs_config.py +++ b/builder/scripts/test_afs_config.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """AFS mount readiness and shared checkpoint profile regression tests.""" +import copy import importlib.util import os from pathlib import Path @@ -24,6 +25,126 @@ def load(name, path): CONFIG = load("afs_config", ROOT / "builder/scripts/prepare-adx-config.py") +class StandaloneCapacityTest(unittest.TestCase): + def test_default_source_is_replaced_after_full_profile_resolution(self): + original = { + "schema_version": 1, + "services": [{"id": "worker", "role": "adxlet", "config": { + "resource_source": {"kind": "auto", "disk_path": "/home/akernel", + "valid_for_seconds": 30}, + "preserved": "value", + }}], + } + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"ENABLE_TRACE": "false"}): + source = ROOT / "builder/config/adx-standalone.yaml" + target = Path(directory) / "resolved.yaml" + with patch.object(CONFIG.subprocess, "run") as run: + run.return_value.stdout = yaml.safe_dump(original) + CONFIG.configure(source, target, None, "unused", sandboxd_capacity=True) + run.assert_called_once_with( + ["/opt/adx/current/bin/adxctl", "--config", str(source), "config", "dump"], + check=True, capture_output=True, text=True, + ) + result = yaml.safe_load(target.read_text()) + expected = copy.deepcopy(original) + expected["services"][0]["config"]["resource_source"] = { + "kind": "sandboxd", "socket": "/var/run/resource.sock", + "valid_for_seconds": 30, + } + self.assertEqual(result, expected) + self.assertEqual(target.stat().st_mode & 0o777, 0o600) + + def test_custom_capacity_sources_are_preserved_without_resolving(self): + sources = ({"resource_source": {"kind": "auto", "disk_path": "/custom", + "valid_for_seconds": 7}}, + {"resource_source": {"kind": "file", "path": "/custom/capacity.json"}}, + {"resource_source": {"kind": "sandboxd", "socket": "/custom.sock", + "valid_for_seconds": 9}}, + {"capacity_file": "/custom/capacity.json"}) + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"ENABLE_TRACE": "false"}): + source = Path(directory) / "custom.yaml" + for explicit in (False, True): + for node in sources: + with self.subTest(explicit=explicit, node=node): + original = {"schema_version": 1} + if explicit: + original["services"] = [{"id": "worker", "role": "adxlet", "config": node}] + else: + original.update({"profile": "standalone", "service_overrides": { + "adxlet": {"config": node}}}) + source.write_text(yaml.safe_dump(original)) + with patch.object(CONFIG.subprocess, "run") as run: + CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) + run.assert_not_called() + self.assertEqual(yaml.safe_load(source.read_text()), original) + + def test_resolution_failure_preserves_in_place_source(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "source.yaml" + original = (ROOT / "builder/config/adx-standalone.yaml").read_text() + source.write_text(original) + with patch.object(CONFIG.subprocess, "run", side_effect=subprocess.CalledProcessError(1, "adxctl")): + with self.assertRaises(subprocess.CalledProcessError): + CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) + self.assertEqual(source.read_text(), original) + + def test_build_time_conversion_does_not_freeze_runtime_expressions(self): + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "source.yaml" + original = yaml.safe_load((ROOT / "builder/config/adx-standalone.yaml").read_text()) + original["namespace"] = "${CUSTOM_NAMESPACE:-akernel}" + source.write_text(yaml.safe_dump(original)) + with patch.object(CONFIG.subprocess, "run") as run: + with self.assertRaisesRegex(ValueError, "static deployment"): + CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) + run.assert_not_called() + self.assertEqual(yaml.safe_load(source.read_text()), original) + + @unittest.skipUnless(os.environ.get("AKERNEL_TEST_ADXCTL"), "requires packaged ADX CLI") + def test_packaged_cli_resolves_default_pvm_trace_and_afs_to_sandboxd(self): + binary = Path(os.environ["AKERNEL_TEST_ADXCTL"]) + source = ROOT / "builder/config/adx-standalone.yaml" + + def resolved(path): + return yaml.safe_load(subprocess.run( + [str(binary), "--config", str(path), "config", "dump"], + check=True, capture_output=True, text=True, + ).stdout) + + for pvm in (False, True): + for trace, mount in ((False, None), (True, None), + (True, Path("/mnt/akernel-shared"))): + with self.subTest(pvm=pvm, trace=trace, mount=mount), tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "prepared.yaml" + baseline = Path(directory) / "baseline.yaml" + # Mock only host mount inspection; the real packaged CLI + # resolves and validates each complete deployment below. + run = subprocess.run + + def mounted(command, **kwargs): + if command[0] == "findmnt": + return subprocess.CompletedProcess(command, 0, "afs-dfs fuse rw\n", "") + return run(command, **kwargs) + + with patch.dict(os.environ, {"ENABLE_TRACE": str(trace).lower()}), patch.object(CONFIG.subprocess, "run", side_effect=mounted): + CONFIG.configure(source, baseline, mount, "test/checkpoints", pvm=pvm) + before = resolved(baseline) + CONFIG.configure(source, target, mount, "test/checkpoints", pvm=pvm, + sandboxd_capacity=True, adxctl=binary) + after = resolved(target) + self.assertNotIn("profile", yaml.safe_load(target.read_text())) + worker = next(service for service in after["services"] if service["role"] == "adxlet") + self.assertEqual(worker["config"].pop("resource_source"), { + "kind": "sandboxd", "socket": "/var/run/resource.sock", + "valid_for_seconds": 30, + }) + old_worker = next(service for service in before["services"] if service["role"] == "adxlet") + self.assertEqual(old_worker["config"].pop("resource_source")["kind"], "auto") + self.assertEqual(after, before) + subprocess.run([str(binary), "--config", str(target), "validate"], + check=True, capture_output=True, text=True) + + class AfsConfigTest(unittest.TestCase): def assert_adx_valid(self, target): binary = os.environ.get("AKERNEL_TEST_ADXCTL") diff --git a/deploy/README.md b/deploy/README.md index 612ef0cf..8aa74ae2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -19,8 +19,8 @@ application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-applicati ## Aggregate sandbox memory budget -The standalone, Helm and both cloud templates enable the cgroup v2 aggregate -sandbox memory policy by default: +Standalone and the direct Helm chart enable the cgroup v2 aggregate sandbox +memory policy by default: ```toml [plugin.resource.memory_limit] @@ -36,11 +36,13 @@ without subtracting the reserve a second time. Helm and cloud Adxlet configs consume that socket. For example, effective memory of 64 GiB and this reserve produce a candidate of 48 GiB; an existing 40 GiB cap remains 40 GiB. -Standalone currently inherits ADX's `auto` capacity source, which observes the -Adxlet's own cgroup rather than the sandbox parent. Its public scheduling -capacity can therefore exceed the parent cap even when `/resource` is correct. -The standalone capacity source must be wired to sandboxd and verified before -relying on the public scheduling capacity to match the new limit. +The bundled standalone ADX profile is resolved at image build time and its +complete capacity source is replaced with `kind: sandboxd`, using +`/var/run/resource.sock`. The PVM, trace and shared-storage variants retain this +source. Public ADX capacity follows the sandbox budget, while allocatable +decreases on resource allocation and recovers on deletion. Custom ADX +configurations with an explicitly selected source remain operator-owned; align +that source with the effective sandbox budget when enabling a parent limit. The default requires cgroup v2 and effective memory greater than 16 GiB, with enough remaining capacity for the requested sandboxes. Docker/OrbStack VM limits @@ -49,16 +51,24 @@ image built with the pinned sandboxd's `total_reserve` support. On a smaller nod explicitly choose a smaller positive `total_reserve` or set `enabled=false` in the sandboxd TOML before starting. Standalone reads its source config template; Helm uses `node.config.sandboxd.config` (`core.node.config.sandboxd.config` in the -umbrella chart). Cloud templates provide the complete TOML and must be adjusted -there or through the existing complete-config override. Values use integer +umbrella chart). Cloud profiles expose the enabled flag and reserve described +below, alongside their existing complete-config override. Values use integer bytes or `B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not accepted. Standard and PVM CI fixtures explicitly use a 2 GiB reserve while keeping the policy enabled. -Verify the operating-system choice for cloud profiles as well. The current -Aliyun `node_pool_image_type="AliyunLinux3"` default is the standard image, -which uses cgroup v1; select and qualify a cgroup v2 image before enabling this -policy. Huawei's selected node image also requires cgroup-mode verification. +Both cloud Terraform profiles keep the policy disabled by default. Their +`sandbox_memory_limit_enabled` input is an explicit opt-in, with +`sandbox_memory_total_reserve="16GiB"`. Guided profiles expose the same inputs: + +```bash +make config SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB +``` + +Select and qualify a cgroup v2 node image before enabling it: the current standard Aliyun +`node_pool_image_type="AliyunLinux3"` default uses cgroup v1, and Huawei's +selected node image also needs cgroup-mode verification. These options do not +change the node operating system or migrate existing nodes to cgroup v2. This reserve is deployment policy, not a measured component budget or a guarantee against host OOM. Distill-fs retains its independent 64 GiB limit and diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index fd9c6859..a24f2980 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -191,7 +191,13 @@ Use XFS for the actual writable filestore and checkpoint paths. In the qualifica The dedicated PVM qualification omits runsc and Kata runtime entries from its temporary configuration and explicitly enables `plugin.runtime.firecracker.virtiofs_enabled=true` for OCI/Nydus directory roots. The maintained standalone default uses EROFS and leaves virtio-fs disabled. A deployment that needs to reproduce this qualification must explicitly select `firecracker-pvm`, enable the compatible virtio-fs profile for directory roots, and configure only the qualified handlers. The original mixed-runtime qualification observed a `runsc list` timeout extending housekeeping beyond its five-second process-health threshold, rejecting PVM reloads. The follow-up sandboxd change polls handlers concurrently with independent deadlines and closes only the failed class's admission/advertisement; successful polls reopen it and deletion remains available. Local storage stalls and other core-module failures still affect process readiness. Mixed-runtime acceptance must distinguish backend inventory failures from shared storage/control-plane pressure. -The PVM image applies `prepare-adx-config.py --pvm` to the shared standalone profile, setting coordinator and adxlet `rpc_timeout_seconds` to 90. Custom/externally supplied ADX configurations must carry these overrides themselves. The ordinary KVM image keeps its original ADX profile. This is a bounded runtime cleanup/reload budget: a recorded cleanup took 30.63 seconds and exceeded the old 30-second limit. The guest-agent startup budget stays at 15 seconds. +Both image profiles resolve the bundled standalone deployment and select +sandboxd's resource socket for scheduling capacity. The PVM image additionally +applies `prepare-adx-config.py --pvm`, setting coordinator and adxlet +`rpc_timeout_seconds` to 90. Custom/externally supplied ADX configurations must +carry these overrides themselves. This is a bounded runtime cleanup/reload +budget: a recorded cleanup took 30.63 seconds and exceeded the old 30-second +limit. The guest-agent startup budget stays at 15 seconds. The maintained PVM CI fixture selects a 2 GiB aggregate sandbox memory reserve for its 8 GiB L1, keeping the memory policy enabled. This explicit qualification diff --git a/deploy/scripts/configure.sh b/deploy/scripts/configure.sh index 208572b7..6a4bea57 100755 --- a/deploy/scripts/configure.sh +++ b/deploy/scripts/configure.sh @@ -31,6 +31,8 @@ image_tag_override="" install_monitor_override="" install_dragonfly_override="" chunk_db_size_override="" +sandbox_memory_limit_enabled_override="" +sandbox_memory_total_reserve_override="" juicefs_enabled="false" juicefs_storage_class="" juicefs_cache_mib="4096" @@ -129,6 +131,14 @@ while [[ $# -gt 0 ]]; do chunk_db_size_override="$2" shift 2 ;; + --sandbox-memory-limit-enabled) + sandbox_memory_limit_enabled_override="$2" + shift 2 + ;; + --sandbox-memory-total-reserve) + sandbox_memory_total_reserve_override="$2" + shift 2 + ;; --juicefs-enabled) juicefs_enabled="$2" shift 2 @@ -285,6 +295,15 @@ set_or_prompt chunk_db_size "Shared ChunkDB capacity (empty uses distill-fs defa if [[ -n "${chunk_db_size}" && ! "${chunk_db_size}" =~ ^[0-9]+(B|KiB|MiB|GiB|TiB)?$ ]]; then die "CHUNK_DB_SIZE must be whole bytes or an integer with B/KiB/MiB/GiB/TiB" fi +set_or_prompt sandbox_memory_limit_enabled \ + "Enable aggregate sandbox memory limit on qualified cgroup v2 nodes (true/false)" \ + "false" "${sandbox_memory_limit_enabled_override}" +set_or_prompt sandbox_memory_total_reserve \ + "Aggregate sandbox memory reserve (effective memory must exceed this when enabled)" \ + "16GiB" "${sandbox_memory_total_reserve_override}" +if [[ ! "${sandbox_memory_total_reserve}" =~ ^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$ ]]; then + die "SANDBOX_MEMORY_TOTAL_RESERVE must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB" +fi set_or_prompt enable_runc "Enable the optional runc runtime (true/false)" "false" "${enable_runc_override}" set_or_prompt schedule_placement_policy \ "YuanRong schedule placement policy (binpack/spread)" "spread" \ @@ -298,6 +317,7 @@ if [[ -z "${grafana_admin_password}" ]]; then fi install_monitor="$(normalize_bool "${install_monitor}")" install_dragonfly="$(normalize_bool "${install_dragonfly}")" +sandbox_memory_limit_enabled="$(normalize_bool "${sandbox_memory_limit_enabled}")" enable_runc="$(normalize_bool "${enable_runc}")" schedule_placement_policy="$(normalize_schedule_placement_policy "${schedule_placement_policy}")" grafana_public_access="$(normalize_bool "${grafana_public_access}")" @@ -389,6 +409,8 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} chunk_db_size = "${chunk_db_size}" +sandbox_memory_limit_enabled = ${sandbox_memory_limit_enabled} +sandbox_memory_total_reserve = "${sandbox_memory_total_reserve}" enable_runc = ${enable_runc} schedule_placement_policy = "${schedule_placement_policy}" EOF @@ -435,6 +457,8 @@ grafana_admin_password = "${grafana_admin_password}" install_dragonfly = ${install_dragonfly} chunk_db_size = "${chunk_db_size}" +sandbox_memory_limit_enabled = ${sandbox_memory_limit_enabled} +sandbox_memory_total_reserve = "${sandbox_memory_total_reserve}" enable_runc = ${enable_runc} schedule_placement_policy = "${schedule_placement_policy}" EOF @@ -458,6 +482,8 @@ CORE_NAMESPACE=akernel MONITOR_NAMESPACE=akernel-monitor INSTALL_DRAGONFLY=${install_dragonfly} CHUNK_DB_SIZE=${chunk_db_size} +SANDBOX_MEMORY_LIMIT_ENABLED=${sandbox_memory_limit_enabled} +SANDBOX_MEMORY_TOTAL_RESERVE=${sandbox_memory_total_reserve} AKERNEL_ENABLE_RUNC=${enable_runc} SCHEDULE_PLACEMENT_POLICY=${schedule_placement_policy} EOF diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 3e9159f4..76f29e28 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -69,6 +69,14 @@ applies to `make standalone-source-test`. See [the memory budget and upgrade contract](../README.md#aggregate-sandbox-memory-budget) before applying the new default to an existing profile. +The bundled ADX configuration takes scheduling capacity from sandboxd's +`/var/run/resource.sock`, including PVM and trace/shared-storage variants. +The image build resolves the default profile before replacing the entire +capacity source. Custom `AKERNEL_ADX_CONFIG` files retain their selected source; +operators enabling the parent limit must align custom capacity reporting with +that limit. Standalone CI checks both the parent/socket budget and public ADX +capacity, plus resource allocation and release after sandbox create/delete. + ### Runtime selection Runsc is the source default. Opt into runc with diff --git a/deploy/terraform/aliyun/README.md b/deploy/terraform/aliyun/README.md index a2759182..6473cddd 100644 --- a/deploy/terraform/aliyun/README.md +++ b/deploy/terraform/aliyun/README.md @@ -10,6 +10,27 @@ It installs: - Optional Dragonfly to `dragonfly_namespace` (controlled by `install_dragonfly`) - Optional prereq `openkruise` to `prereq_namespace` (controlled by `install_prereqs`) +## Aggregate sandbox memory budget + +The cloud profile leaves the parent memory policy disabled by default and +retains its existing OS selection. Standard `AliyunLinux3` uses cgroup v1; +the policy requires cgroup v2. Before opting in, select and qualify a v2 node +image, verify the actual node/container cgroup mode, and size the reserve for +that profile. The module does not convert an existing node's cgroup mode. + +Set `sandbox_memory_limit_enabled=true` and +`sandbox_memory_total_reserve="16GiB"` in `terraform.tfvars`, or pass +`SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB` to +`make config`. Effective memory, including finite ancestors, must exceed the +reserve and leave room for sandbox requests. Smaller nodes require an explicit +smaller positive reserve. These settings also apply when reusing a cluster; +complete TOML overrides remain responsible for their own memory policy. + +Adding or lowering a parent cap requires drained sandboxes. The reserve is +deployment policy rather than a host-OOM guarantee; distill-fs has an independent +limit. See [the memory budget and upgrade contract](../../README.md#aggregate-sandbox-memory-budget) +before enabling the policy on existing nodes. + ## Pod PID budget `node_pool_pids_limit` defaults to `1620780` for the default pool and all diff --git a/deploy/terraform/aliyun/main.tf b/deploy/terraform/aliyun/main.tf index 7b452a56..16822691 100644 --- a/deploy/terraform/aliyun/main.tf +++ b/deploy/terraform/aliyun/main.tf @@ -135,6 +135,9 @@ locals { node_image_repo = length(var.node_image_repository) > 0 ? var.node_image_repository : "${local.acr_registry}/all-in-one" core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { + sandbox_memory_limit_enabled = var.sandbox_memory_limit_enabled + sandbox_memory_total_reserve = var.sandbox_memory_total_reserve + cluster_name = var.cluster_name adx_public_service_enabled = var.adx_public_service_enabled adx_service_type = var.adx_service_type diff --git a/deploy/terraform/aliyun/terraform.tfvars.example b/deploy/terraform/aliyun/terraform.tfvars.example index 45ebcd1a..ab59f2b5 100644 --- a/deploy/terraform/aliyun/terraform.tfvars.example +++ b/deploy/terraform/aliyun/terraform.tfvars.example @@ -142,6 +142,11 @@ node_secret_create = true # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Explicit opt-in after qualifying cgroup v2 and the effective memory budget. +# Standard AliyunLinux3 defaults to cgroup v1; select a qualified v2 node image. +sandbox_memory_limit_enabled = false +sandbox_memory_total_reserve = "16GiB" + # AKernel defaults YuanRong placement to "spread"; use "binpack" to compact. schedule_placement_policy = "spread" diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 5b3c2e25..393f6b91 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -165,9 +165,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = true - # Requires effective memory above this reserve; tune for the deployment. - total_reserve = "16GiB" + # Explicit opt-in requires qualified cgroup v2 nodes and sufficient memory. + enabled = ${sandbox_memory_limit_enabled} + total_reserve = ${jsonencode(sandbox_memory_total_reserve)} [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/aliyun/variables.tf b/deploy/terraform/aliyun/variables.tf index 0f04906d..525d626a 100644 --- a/deploy/terraform/aliyun/variables.tf +++ b/deploy/terraform/aliyun/variables.tf @@ -808,6 +808,22 @@ variable "tempo_resources" { default = {} } +variable "sandbox_memory_limit_enabled" { + type = bool + description = "Opt into the aggregate sandbox parent memory limit only on qualified cgroup v2 nodes. Drain before adding or lowering a cap." + default = false +} + +variable "sandbox_memory_total_reserve" { + type = string + description = "Memory excluded from the aggregate sandbox budget when enabled; effective node memory must exceed this deployment-specific reserve." + default = "16GiB" + validation { + condition = can(regex("^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$", var.sandbox_memory_total_reserve)) + error_message = "sandbox_memory_total_reserve must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB." + } +} + variable "chunk_db_size" { type = string description = "Shared distill-fs ChunkDB map capacity, e.g. 64GiB; empty uses the Linux default of 100 GiB. Existing caches must not be resized." diff --git a/deploy/terraform/huaweicloud/README.md b/deploy/terraform/huaweicloud/README.md index a6d5bd43..784cbdf6 100644 --- a/deploy/terraform/huaweicloud/README.md +++ b/deploy/terraform/huaweicloud/README.md @@ -6,6 +6,27 @@ contract as the Aliyun module: one all-in-one AKernel image, an Agent DX identity Secret, managed Redis, a direct ADX public Service, optional public Grafana, and local state under `.akernel//`. +## Aggregate sandbox memory budget + +The cloud profile leaves the parent memory policy disabled by default and +retains its existing OS selection. Before opting in, qualify cgroup v2 on the +actual CCE node image and inside the AKernel container, then size the reserve +for that profile. The OS name alone does not establish the cgroup mode, and +the module does not convert existing nodes to cgroup v2. + +Set `sandbox_memory_limit_enabled=true` and +`sandbox_memory_total_reserve="16GiB"` in `terraform.tfvars`, or pass +`SANDBOX_MEMORY_LIMIT_ENABLED=true SANDBOX_MEMORY_TOTAL_RESERVE=16GiB` to +`make config`. Effective memory, including finite ancestors, must exceed the +reserve and leave room for sandbox requests. Smaller nodes require an explicit +smaller positive reserve. These settings also apply when reusing a cluster; +complete TOML overrides remain responsible for their own memory policy. + +Adding or lowering a parent cap requires drained sandboxes. The reserve is +deployment policy rather than a host-OOM guarantee; distill-fs has an independent +limit. See [the memory budget and upgrade contract](../../README.md#aggregate-sandbox-memory-budget) +before enabling the policy on existing nodes. + ## Pod PID budget [CCE documents `pod-pids-limit=-1` by default](https://support.huaweicloud.com/intl/en-us/usermanual-cce/cce_10_0652.html), diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index 2626e66a..39c227a4 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -87,6 +87,9 @@ locals { ]) : local.node_pool_bootstrap_script core_values = templatefile("${path.module}/values-akernel.yaml.tmpl", { + sandbox_memory_limit_enabled = var.sandbox_memory_limit_enabled + sandbox_memory_total_reserve = var.sandbox_memory_total_reserve + cluster_name = var.cluster_name adx_public_service_enabled = var.adx_public_service_enabled adx_service_type = var.adx_service_type diff --git a/deploy/terraform/huaweicloud/terraform.tfvars.example b/deploy/terraform/huaweicloud/terraform.tfvars.example index 7fff9e24..7b4de2b0 100644 --- a/deploy/terraform/huaweicloud/terraform.tfvars.example +++ b/deploy/terraform/huaweicloud/terraform.tfvars.example @@ -88,6 +88,11 @@ node_image_tag = "" # matching AKERNEL_ENABLE_RUNC=true profile setting. enable_runc = false +# Explicit opt-in after qualifying the actual node image's cgroup v2 mode +# and effective memory budget; the OS name alone does not establish support. +sandbox_memory_limit_enabled = false +sandbox_memory_total_reserve = "16GiB" + # AKernel defaults YuanRong placement to "spread"; use "binpack" to compact. schedule_placement_policy = "spread" diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index ed47ff35..6696a174 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -142,9 +142,9 @@ node: enable_cache_in_stock_mode=true [plugin.resource.memory_limit] - enabled = true - # Requires effective memory above this reserve; tune for the deployment. - total_reserve = "16GiB" + # Explicit opt-in requires qualified cgroup v2 nodes and sufficient memory. + enabled = ${sandbox_memory_limit_enabled} + total_reserve = ${jsonencode(sandbox_memory_total_reserve)} [plugin.runtime] image_lib_dir="/home/akernel/images" diff --git a/deploy/terraform/huaweicloud/variables.tf b/deploy/terraform/huaweicloud/variables.tf index bf2e29f1..655a2d75 100644 --- a/deploy/terraform/huaweicloud/variables.tf +++ b/deploy/terraform/huaweicloud/variables.tf @@ -830,6 +830,22 @@ variable "tempo_resources" { default = {} } +variable "sandbox_memory_limit_enabled" { + type = bool + description = "Opt into the aggregate sandbox parent memory limit only on qualified cgroup v2 nodes. Drain before adding or lowering a cap." + default = false +} + +variable "sandbox_memory_total_reserve" { + type = string + description = "Memory excluded from the aggregate sandbox budget when enabled; effective node memory must exceed this deployment-specific reserve." + default = "16GiB" + validation { + condition = can(regex("^0*[1-9][0-9]*(B|KiB|MiB|GiB|TiB)?$", var.sandbox_memory_total_reserve)) + error_message = "sandbox_memory_total_reserve must be positive whole bytes or an integer with B/KiB/MiB/GiB/TiB." + } +} + variable "chunk_db_size" { type = string description = "Shared distill-fs ChunkDB map capacity, e.g. 64GiB; empty uses the Linux default of 100 GiB. Existing caches must not be resized." diff --git a/deploy/terraform/test_adx_contract.py b/deploy/terraform/test_adx_contract.py index 5feae04e..ad4ffae2 100755 --- a/deploy/terraform/test_adx_contract.py +++ b/deploy/terraform/test_adx_contract.py @@ -5,6 +5,7 @@ from __future__ import annotations import json +import os import re import shutil import subprocess @@ -15,6 +16,164 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent +MEMORY_INPUTS = ("sandbox_memory_limit_enabled", "sandbox_memory_total_reserve") + + +def sandboxd_config(values): + block = re.search(r"(?ms)^ config: \|-\n(.*?)(?=^ {0,6}\S|\Z)", values) + if block is None: + raise AssertionError("sandboxd TOML missing from generated cloud values") + return tomllib.loads(textwrap.dedent(block.group(1))) + + +@unittest.skipUnless(shutil.which("terraform"), "Terraform is required for template evaluation") +class SandboxMemoryPolicyValidationTest(unittest.TestCase): + def render(self, provider, inputs): + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) + variables = (ROOT / provider / "variables.tf").read_text() + blocks = [ + re.search(rf'(?ms)^variable "{name}" {{.*?^}}', variables).group() + for name in MEMORY_INPUTS + ] + main = (ROOT / provider / "main.tf").read_text() + mapping = re.search( + r'(?ms)^ core_values = templatefile\("\$\{path.module\}/values-akernel.yaml.tmpl", \{.*?^ \}\)', + main, + ).group() + # Keep the production templatefile call and memory-input wiring. + # Other deployment inputs use inert fixtures: no cloud providers, + # credentials, node pools or Helm releases participate in the plan. + booleans = { + "acr_secret_enabled", "enable_kruise", "install_monitor", + "adx_public_service_enabled", "enable_runc", "native_filestore", + "node_home_use_csi_ephemeral", "node_secret_create", + } + + def fixture_input(match): + key = match.group(1) + if key in MEMORY_INPUTS: + return match.group() + value = False if key in booleans else "" + return f" {key} = {json.dumps(value)}" + + mapping = re.sub(r"(?m)^ (\w+)\s+= .*?$", fixture_input, mapping) + blocks.append("locals {\n" + mapping + "\n}") + blocks.append('output "rendered" { value = local.core_values }') + (fixture / "main.tf").write_text("\n".join(blocks)) + shutil.copy2(ROOT / provider / "values-akernel.yaml.tmpl", fixture) + (fixture / "inputs.tfvars.json").write_text(json.dumps(inputs)) + subprocess.run( + ["terraform", "init", "-backend=false", "-input=false", "-no-color"], + cwd=fixture, capture_output=True, text=True, check=True, + ) + result = subprocess.run( + ["terraform", "plan", "-input=false", "-lock=false", "-refresh=false", + "-no-color", "-var-file=inputs.tfvars.json", "-out=plan"], + cwd=fixture, capture_output=True, text=True, + ) + if result.returncode: + return result, None + plan = subprocess.run( + ["terraform", "show", "-json", "plan"], + cwd=fixture, capture_output=True, text=True, check=True, + ) + rendered = json.loads(plan.stdout)["planned_values"]["outputs"]["rendered"]["value"] + return result, sandboxd_config(rendered) + + def test_cloud_defaults_and_explicit_memory_policy_overrides(self): + for provider in ("aliyun", "huaweicloud"): + for inputs, enabled, reserve in ( + ({}, False, "16GiB"), + ({"sandbox_memory_limit_enabled": True}, True, "16GiB"), + ({"sandbox_memory_limit_enabled": True, "sandbox_memory_total_reserve": "2GiB"}, True, "2GiB"), + ({"sandbox_memory_limit_enabled": False, "sandbox_memory_total_reserve": "2GiB"}, False, "2GiB"), + ): + with self.subTest(provider=provider, inputs=inputs): + result, config = self.render(provider, inputs) + self.assertEqual(result.returncode, 0, result.stderr) + resource = config["plugin"]["resource"] + self.assertEqual(resource["memory_limit"], {"enabled": enabled, "total_reserve": reserve}) + self.assertIs(resource["memory_limit"]["enabled"], enabled) + self.assertEqual(resource["cgroup_root_name"], "/akernel") + self.assertEqual(resource["max_instance_num"], 1000) + self.assertIs(resource["enable_cache_in_stock_mode"], True) + self.assertEqual(config["plugin"]["image"]["cgroup_memory_limit"], "64GiB") + + def test_invalid_reserves_fail_before_deployment(self): + for provider in ("aliyun", "huaweicloud"): + for reserve in ("", "0", "0GiB", "16G", "auto", "-1GiB", "16GiB\"\nenabled=true"): + with self.subTest(provider=provider, reserve=reserve): + result, _ = self.render(provider, {"sandbox_memory_total_reserve": reserve}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("sandbox_memory_total_reserve", result.stderr) + + +class GuidedMemoryPolicyTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.fixture = Path(self.temporary.name) + scripts = self.fixture / "deploy/scripts" + scripts.mkdir(parents=True) + for name in ("configure.sh", "common.sh", "shared_storage_config.py"): + shutil.copy2(ROOT.parent / "scripts" / name, scripts) + shutil.copy2(ROOT.parents[1] / "Makefile", self.fixture) + commands = self.fixture / "bin" + commands.mkdir() + formatter = commands / "terraform" + # Profile generation only calls terraform fmt; evaluated rendering and + # validation are covered separately with the real Terraform CLI above. + formatter.write_text('#!/bin/sh\n[ "$1" = fmt ]\n') + formatter.chmod(0o755) + self.environment = dict(os.environ, PATH=str(commands) + os.pathsep + os.environ["PATH"]) + for name in ("SANDBOX_MEMORY_LIMIT_ENABLED", "SANDBOX_MEMORY_TOTAL_RESERVE", "FORCE"): + self.environment.pop(name, None) + + def generate(self, provider, name, *settings): + return subprocess.run( + ["make", "config", f"VENDOR={provider}", f"ENV={name}", "NON_INTERACTIVE=1", + "NODE_POOL_KEY_NAME=fixture", "IMAGE_REPOSITORY=example.invalid/fixture", + "IMAGE_TAG=fixture", *settings], + cwd=self.fixture, env=self.environment, capture_output=True, text=True, + ) + + def test_generated_profiles_default_to_opt_out_and_preserve_explicit_opt_in(self): + for provider in ("aliyun", "huaweicloud"): + for name, settings, enabled, reserve in ( + ("default", (), "false", "16GiB"), + ("enabled", ("SANDBOX_MEMORY_LIMIT_ENABLED=true",), "true", "16GiB"), + ("small", ("SANDBOX_MEMORY_LIMIT_ENABLED=true", "SANDBOX_MEMORY_TOTAL_RESERVE=2GiB"), "true", "2GiB"), + ): + profile_name = f"{provider}-{name}" + with self.subTest(provider=provider, profile=name): + result = self.generate(provider, profile_name, *settings) + self.assertEqual(result.returncode, 0, result.stderr) + profile = self.fixture / ".akernel" / profile_name + tfvars = (profile / "terraform.tfvars").read_text() + config = (profile / "config.env").read_text() + self.assertIn(f"sandbox_memory_limit_enabled = {enabled}\n", tfvars) + self.assertIn(f'sandbox_memory_total_reserve = "{reserve}"\n', tfvars) + self.assertIn(f"SANDBOX_MEMORY_LIMIT_ENABLED={enabled}\n", config) + self.assertIn(f"SANDBOX_MEMORY_TOTAL_RESERVE={reserve}\n", config) + + def test_invalid_input_and_existing_profiles_are_not_replaced(self): + for provider in ("aliyun", "huaweicloud"): + with self.subTest(provider=provider): + result = self.generate(provider, provider) + self.assertEqual(result.returncode, 0, result.stderr) + profile = self.fixture / ".akernel" / provider + original = {name: (profile / name).read_bytes() for name in ("config.env", "terraform.tfvars")} + for settings in ( + ("SANDBOX_MEMORY_LIMIT_ENABLED=maybe",), + ("SANDBOX_MEMORY_TOTAL_RESERVE=0GiB",), + ("SANDBOX_MEMORY_TOTAL_RESERVE=16G",), + ("SANDBOX_MEMORY_LIMIT_ENABLED=true",), + ): + result = self.generate(provider, provider, *settings) + self.assertNotEqual(result.returncode, 0) + for name, data in original.items(): + self.assertEqual((profile / name).read_bytes(), data) @unittest.skipUnless( @@ -178,21 +337,26 @@ def test_managed_clients_cannot_target_non_linux_nodes(self): class AdxTerraformContractTest(unittest.TestCase): - def test_each_provider_enables_parent_limit_with_16gib_total_reserve(self): + def test_cloud_memory_policy_is_explicitly_wired_and_defaults_off(self): for provider in ("aliyun", "huaweicloud"): with self.subTest(provider=provider): values = (ROOT / provider / "values-akernel.yaml.tmpl").read_text() - block = re.search( - r"(?ms)^ config: \|-\n(.*?)(?=^ {0,6}\S|\Z)", values - ) - self.assertIsNotNone(block) - config = tomllib.loads(textwrap.dedent(block.group(1))) + main = (ROOT / provider / "main.tf").read_text() + variables = (ROOT / provider / "variables.tf").read_text() + for name, default in zip(MEMORY_INPUTS, ("false", '"16GiB"')): + self.assertRegex(main, rf"{name}\s*= var\.{name}") + variable = re.search(rf'(?ms)^variable "{name}" {{.*?^}}', variables).group() + self.assertRegex(variable, rf"default\s*= {default}") + self.assertIn("enabled = ${sandbox_memory_limit_enabled}", values) + self.assertIn("total_reserve = ${jsonencode(sandbox_memory_total_reserve)}", values) + config = sandboxd_config(values.replace("${sandbox_memory_limit_enabled}", "false").replace( + "${jsonencode(sandbox_memory_total_reserve)}", '"16GiB"')) resource = config["plugin"]["resource"] self.assertEqual( resource["memory_limit"], - {"enabled": True, "total_reserve": "16GiB"}, + {"enabled": False, "total_reserve": "16GiB"}, ) - self.assertIs(resource["memory_limit"]["enabled"], True) + self.assertIs(resource["memory_limit"]["enabled"], False) self.assertEqual(resource["cgroup_root_name"], "/akernel") self.assertEqual(resource["max_instance_num"], 1000) self.assertIs(resource["enable_cache_in_stock_mode"], True) diff --git a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py index 8216a91c..f4badecc 100644 --- a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py +++ b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py @@ -16,6 +16,7 @@ import os import tempfile +import time import unittest import uuid from pathlib import Path @@ -29,6 +30,55 @@ ) _RUNTIME = os.environ.get("AKERNEL_TEST_RUNTIME", "runsc") _IMAGE = os.environ.get("AKERNEL_TEST_IMAGE") or None +_MEMORY_BUDGET = os.environ.get("AKERNEL_TEST_EXPECT_MEMORY_BYTES") + + +@unittest.skipUnless( + _ENABLED and _MEMORY_BUDGET, + "set the SDK integration environment and AKERNEL_TEST_EXPECT_MEMORY_BYTES", +) +class MemoryBudgetIntegrationTest(unittest.TestCase): + """Check the public scheduler against a measured single-node parent cap.""" + + def test_capacity_and_allocation_follow_the_parent_memory_budget(self): + expected = int(_MEMORY_BUDGET) / (1024 * 1024) + self.assertGreater(expected, 0) + + def wait_for_memory(allocatable=None): + deadline = time.monotonic() + 60 + last = [] + while time.monotonic() < deadline: + last = resources() + self.assertLessEqual(len(last), 1, "requires an isolated single node") + if last: + node = last[0] + capacity = node.capacity.get("Memory", -1) + available = node.allocatable.get("Memory", -1) + if abs(capacity - expected) < 0.000001 and ( + allocatable is None + or abs(available - allocatable) < 0.000001 + ): + return node + time.sleep(0.5) + self.fail( + f"expected memory capacity={expected}, allocatable={allocatable}; " + f"last resource view: {last}" + ) + + before = wait_for_memory() + available = before.allocatable["Memory"] + self.assertGreaterEqual(available, 1024) + sandbox = Sandbox(cpu=1000, memory=1024, runtime=_RUNTIME, image=_IMAGE) + try: + self.assertEqual( + sandbox.commands.run("printf memory-budget").stdout, "memory-budget" + ) + running = wait_for_memory(available - 1024) + self.assertEqual(running.id, before.id) + finally: + sandbox.kill() + released = wait_for_memory(available) + self.assertEqual(released.id, before.id) @unittest.skipUnless( From eebf9432e6ddcf8900558979d9d6da25019176b0 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 11:08:56 +0800 Subject: [PATCH 4/9] test(deploy): cover resource units and current qualification inputs Match public whole-MiB memory reporting and exercise fractional CPU allocation in isolated qualification. Pass the measured parent cap to PVM tests and retain strict skipped-test checks. Keep the shared ACK fixture importable on Python 3.10, and include its current policy file and boolean input in the cloud fixtures. Signed-off-by: wanghongtao.wangho --- deploy/pvm/run-adx-ci.sh | 6 ++ deploy/pvm/test_host_profile.py | 42 ++++++++++ deploy/terraform/test_adx_contract.py | 9 +- .../e2e/standalone/test_sdk_contracts.py | 33 ++++++-- .../unit/test_e2e_memory_budget_fixture.py | 84 +++++++++++++++++++ 5 files changed, 166 insertions(+), 8 deletions(-) create mode 100644 sdk/python/tests/unit/test_e2e_memory_budget_fixture.py diff --git a/deploy/pvm/run-adx-ci.sh b/deploy/pvm/run-adx-ci.sh index 7fd12e4c..10c7f6c4 100644 --- a/deploy/pvm/run-adx-ci.sh +++ b/deploy/pvm/run-adx-ci.sh @@ -134,6 +134,12 @@ RESOLVER python3 deploy/pvm/dns-fixture.py > "${out}/dns-fixture.log" 2>&1 & printf '%s\n' "$!" > "${out}/dns-fixture.pid" bash deploy/standalone/start.sh > "${out}/standalone-start.log" 2>&1 +AKERNEL_TEST_EXPECT_MEMORY_BYTES=$(docker exec akernel-node cat /sys/fs/cgroup/akernel/memory.max) +[[ "${AKERNEL_TEST_EXPECT_MEMORY_BYTES}" =~ ^[1-9][0-9]*$ ]] || { + echo 'PVM qualification requires a finite positive sandbox parent memory limit' >&2 + exit 1 +} +export AKERNEL_TEST_EXPECT_MEMORY_BYTES export AKERNEL_TEST_NETWORK_BLOCKED_HOST=blocked.pvm-ci.test export AKERNEL_TEST_NETWORK_ALLOWED_HOST=allowed.pvm-ci.test export AKERNEL_TEST_NETWORK_WILDCARD_HOST=child.blocked.pvm-ci.test diff --git a/deploy/pvm/test_host_profile.py b/deploy/pvm/test_host_profile.py index 0f9aea7a..f7e8587f 100644 --- a/deploy/pvm/test_host_profile.py +++ b/deploy/pvm/test_host_profile.py @@ -2,6 +2,7 @@ """Check the maintained profile and Bash compatibility exports without builds.""" import hashlib import json +import os from pathlib import Path import subprocess import unittest @@ -45,5 +46,46 @@ def test_shell_exports_are_derived_from_the_json_profile(self): self.assertEqual(result.stdout.splitlines(), list(expected.values())) +class CiMemoryBudgetTest(unittest.TestCase): + def measured_budget(self, value, docker_status=0): + script = (ROOT / "deploy/pvm/run-adx-ci.sh").read_text() + after_start = script.split( + 'bash deploy/standalone/start.sh > "${out}/standalone-start.log" 2>&1\n', 1 + )[1] + preparation = after_start.split("export AKERNEL_TEST_NETWORK_BLOCKED_HOST=", 1)[0] + environment = dict(os.environ) + environment.pop("AKERNEL_TEST_EXPECT_MEMORY_BYTES", None) + environment.update(MEASURED_MEMORY_BYTES=value, DOCKER_STATUS=str(docker_status)) + return subprocess.run( + ["bash", "-euc", ''' +docker() { + [[ "$*" == "exec akernel-node cat /sys/fs/cgroup/akernel/memory.max" ]] || return 99 + printf '%s\\n' "${MEASURED_MEMORY_BYTES}" + return "${DOCKER_STATUS}" +} +''' + preparation + ''' +python3 -c 'import os; print(os.environ["AKERNEL_TEST_EXPECT_MEMORY_BYTES"])' +'''], + text=True, capture_output=True, env=environment, timeout=10, + ) + + def test_pvm_sdk_tests_inherit_the_measured_parent_limit(self): + maximum = str(6 * 1024**3 + 4096) + result = self.measured_budget(maximum) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), maximum) + + def test_pvm_qualification_rejects_missing_or_unlimited_parent_caps(self): + for value in ("max", "0", "-1", ""): + with self.subTest(value=value): + result = self.measured_budget(value) + self.assertNotEqual(result.returncode, 0) + self.assertIn("finite positive sandbox parent memory limit", result.stderr) + + def test_pvm_qualification_preserves_parent_probe_failures(self): + result = self.measured_budget(str(6 * 1024**3), docker_status=17) + self.assertEqual(result.returncode, 17) + + if __name__ == "__main__": unittest.main() diff --git a/deploy/terraform/test_adx_contract.py b/deploy/terraform/test_adx_contract.py index 06e36b7d..d46e0d3e 100755 --- a/deploy/terraform/test_adx_contract.py +++ b/deploy/terraform/test_adx_contract.py @@ -11,7 +11,6 @@ import subprocess import tempfile import textwrap -import tomllib import unittest from pathlib import Path @@ -20,6 +19,9 @@ def sandboxd_config(values): + # ACK's Python 3.10 tests share evaluate_fixture without parsing TOML. + import tomllib + block = re.search(r"(?ms)^ config: \|-\n(.*?)(?=^ {0,6}\S|\Z)", values) if block is None: raise AssertionError("sandboxd TOML missing from generated cloud values") @@ -47,7 +49,7 @@ def render(self, provider, inputs): booleans = { "acr_secret_enabled", "enable_kruise", "install_monitor", "adx_public_service_enabled", "enable_runc", "native_filestore", - "node_home_use_csi_ephemeral", "node_secret_create", + "node_home_use_csi_ephemeral", "node_secret_create", "node_image_pull_auto", } def fixture_input(match): @@ -119,6 +121,9 @@ def setUp(self): for name in ("configure.sh", "common.sh", "shared_storage_config.py"): shutil.copy2(ROOT.parent / "scripts" / name, scripts) shutil.copy2(ROOT.parents[1] / "Makefile", self.fixture) + policy = self.fixture / "deploy/terraform/aliyun/node-image-pull-policy.json" + policy.parent.mkdir(parents=True) + shutil.copy2(ROOT / "aliyun" / policy.name, policy) commands = self.fixture / "bin" commands.mkdir() formatter = commands / "terraform" diff --git a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py index f4badecc..cbc7552d 100644 --- a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py +++ b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py @@ -41,10 +41,15 @@ class MemoryBudgetIntegrationTest(unittest.TestCase): """Check the public scheduler against a measured single-node parent cap.""" def test_capacity_and_allocation_follow_the_parent_memory_budget(self): - expected = int(_MEMORY_BUDGET) / (1024 * 1024) + # The public resource API rounds byte capacities down to whole MiB. + expected = int(_MEMORY_BUDGET) // (1024 * 1024) self.assertGreater(expected, 0) + cpu_budget = os.environ.get("AKERNEL_TEST_EXPECT_CPU_MILLIS") + expected_cpu = int(cpu_budget) if cpu_budget else None + request_cpu = min(1000, expected_cpu) if expected_cpu is not None else 1000 + self.assertGreater(request_cpu, 0) - def wait_for_memory(allocatable=None): + def wait_for_memory(allocatable=None, allocatable_cpu=None): deadline = time.monotonic() + 60 last = [] while time.monotonic() < deadline: @@ -54,30 +59,46 @@ def wait_for_memory(allocatable=None): node = last[0] capacity = node.capacity.get("Memory", -1) available = node.allocatable.get("Memory", -1) + cpu_matches = expected_cpu is None or ( + node.capacity.get("CPU", -1) == expected_cpu + and ( + allocatable_cpu is None + or node.allocatable.get("CPU", -1) == allocatable_cpu + ) + ) if abs(capacity - expected) < 0.000001 and ( allocatable is None or abs(available - allocatable) < 0.000001 - ): + ) and cpu_matches: return node time.sleep(0.5) self.fail( f"expected memory capacity={expected}, allocatable={allocatable}; " + f"CPU capacity={expected_cpu}, allocatable={allocatable_cpu}; " f"last resource view: {last}" ) before = wait_for_memory() available = before.allocatable["Memory"] self.assertGreaterEqual(available, 1024) - sandbox = Sandbox(cpu=1000, memory=1024, runtime=_RUNTIME, image=_IMAGE) + available_cpu = ( + before.allocatable["CPU"] if expected_cpu is not None else None + ) + if available_cpu is not None: + self.assertGreaterEqual(available_cpu, request_cpu) + sandbox = Sandbox(cpu=request_cpu, memory=1024, runtime=_RUNTIME, image=_IMAGE) try: self.assertEqual( sandbox.commands.run("printf memory-budget").stdout, "memory-budget" ) - running = wait_for_memory(available - 1024) + running = wait_for_memory( + available - 1024, + available_cpu - request_cpu if available_cpu is not None else None, + ) self.assertEqual(running.id, before.id) finally: sandbox.kill() - released = wait_for_memory(available) + released = wait_for_memory(available, available_cpu) self.assertEqual(released.id, before.id) diff --git a/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py b/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py new file mode 100644 index 00000000..1a5c7873 --- /dev/null +++ b/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py @@ -0,0 +1,84 @@ +"""Public resource units used by the standalone memory-budget regression.""" + +import os +import unittest +from unittest.mock import patch + +from akernel_sdk import NodeInfo +from tests.e2e.standalone import test_sdk_contracts as fixture + + +class MemoryBudgetFixtureTest(unittest.TestCase): + def test_page_aligned_parent_caps_follow_whole_mib_allocation_and_release(self): + public_capacity = 6144.0 + cases = [(extra, None) for extra in (0, 4096, 1024 * 1024 - 4096)] + cases.extend([(4096, 500), (4096, 1500)]) + for extra_bytes, cpu_budget in cases: + with self.subTest(extra_bytes=extra_bytes, cpu_budget=cpu_budget): + maximum = 6 * 1024**3 + extra_bytes + request_cpu = min(1000, cpu_budget) if cpu_budget is not None else 1000 + + def node(available, cpu=None, budget=cpu_budget): + return NodeInfo( + id="single-node", + status=0, + capacity={ + "Memory": public_capacity, + **({"CPU": budget} if budget is not None else {}), + }, + allocatable={ + "Memory": available, + **({"CPU": cpu} if cpu is not None else {}), + }, + labels={}, + ) + + case = fixture.MemoryBudgetIntegrationTest( + "test_capacity_and_allocation_follow_the_parent_memory_budget" + ) + with ( + patch.object(fixture, "_MEMORY_BUDGET", str(maximum)), + patch.dict( + os.environ, + { + "AKERNEL_TEST_EXPECT_CPU_MILLIS": + str(cpu_budget) if cpu_budget else "" + }, + ), + patch.object( + fixture, + "resources", + side_effect=[ + [node(public_capacity, cpu_budget)], + [node( + public_capacity - 1024, + cpu_budget - request_cpu if cpu_budget else None, + )], + [node(public_capacity, cpu_budget)], + ], + ) as resources, + patch.object(fixture, "Sandbox") as sandbox, + patch.object( + fixture.time, + "sleep", + side_effect=AssertionError( + "public memory should match immediately" + ), + ), + ): + command = sandbox.return_value.commands.run.return_value + command.stdout = "memory-budget" + case.test_capacity_and_allocation_follow_the_parent_memory_budget() + + self.assertEqual(resources.call_count, 3) + sandbox.assert_called_once_with( + cpu=request_cpu, + memory=1024, + runtime=fixture._RUNTIME, + image=fixture._IMAGE, + ) + sandbox.return_value.kill.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() From d676b67a1e27dab2a321b1971b64866a03c6cddf Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 11:48:25 +0800 Subject: [PATCH 5/9] docs(deploy): clarify resource qualification units Document whole-MiB public memory reporting and the optional CPU budget used by isolated qualification. Retain byte-level parent and socket checks alongside public allocation and release assertions. Signed-off-by: wanghongtao.wangho --- AGENTS.md | 3 +++ deploy/README.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 07713de7..f0e8e497 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -741,6 +741,9 @@ The bounded standalone CI profile selects a 2 GiB reserve and exports the measured parent limit as `AKERNEL_TEST_EXPECT_MEMORY_BYTES`. The SDK E2E regression checks public capacity plus allocation and release against that single-node budget; retain the parent/socket check as a separate assertion. +Public memory values round down to whole MiB. Keep page-aligned, non-MiB parent +fixtures and optional `AKERNEL_TEST_EXPECT_CPU_MILLIS` qualification coverage so +fractional CPU budgets can be exercised with schedulable requests. CI explicitly trusts the generated standalone public certificate through `SSL_CERT_FILE` and enables `AKERNEL_VERIFY_TLS=1` before SDK examples and diff --git a/deploy/README.md b/deploy/README.md index 1618068d..51efff3b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -43,6 +43,9 @@ source. Public ADX capacity follows the sandbox budget, while allocatable decreases on resource allocation and recovers on deletion. Custom ADX configurations with an explicitly selected source remain operator-owned; align that source with the effective sandbox budget when enabling a parent limit. +The public API reports memory in whole MiB rounded down; parent cgroups and the +sandboxd socket retain byte precision. Compare each interface in its documented +units, including when a finite ancestor is only page-aligned. The default requires cgroup v2 and effective memory greater than 16 GiB, with enough remaining capacity for the requested sandboxes. Docker/OrbStack VM limits From fa19bfa7e7d1fc244a7cb75795b3480a43cdf815 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 16:59:40 +0800 Subject: [PATCH 6/9] fix(deploy): defer unqualified PVM memory enforcement Keep the aggregate parent policy disabled for PVM unless an operator explicitly enables it and qualifies the selected reserve. Retain the ordinary KVM defaults and keep PVM smoke tests free of unsupported budget qualification claims. Advance sandboxd to preserve millicores in its resource observations. The matching ADX consumer candidate is qualified through release overrides; its formal archive pins must be published before integration. Signed-off-by: wanghongtao.wangho --- AGENTS.md | 7 ++- deploy/README.md | 16 +++++- .../charts/core/templates/node/configmap.yaml | 11 ++++ .../tests/test_shared_checkpoint_storage.py | 45 +++++++++++++++ deploy/akernel/charts/core/values.yaml | 3 + deploy/pvm-runtime.md | 13 +++-- deploy/pvm/run-adx-ci.sh | 15 ++--- deploy/pvm/test_host_profile.py | 56 +++++++------------ deploy/standalone/README.md | 11 +++- deploy/standalone/start.sh | 26 +++++++++ deploy/standalone/test_start_config.py | 37 ++++++++++++ deploy/standalone/test_start_platform.py | 7 +++ sdk/python/tests/e2e/README.md | 8 +++ sdk/python/tests/e2e/run.py | 2 +- src/sandboxd | 2 +- 15 files changed, 201 insertions(+), 58 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7d00b550..ed69f2e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -241,7 +241,12 @@ distill-fs release's packaged manifest. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and Terraform-based cloud provisioning. Standalone and the direct Helm chart enable the cgroup v2 parent memory policy -with `total_reserve="16GiB"`. Both cloud Terraform profiles leave it disabled +with `total_reserve="16GiB"` for the ordinary KVM profile. PVM keeps it disabled +until explicitly selected with `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` in +standalone or `node.config.sandboxd.pvmMemoryLimitEnabled=true` in Helm. Size +the reserve for the selected L1 and qualify that finite-cap combination on the +matching PVM host before rollout; ordinary runsc tests do not qualify PVM. +Both cloud Terraform profiles leave it disabled until explicitly enabled with `sandbox_memory_limit_enabled`; their default operating-system images have not been qualified for this policy. Guided profiles expose `SANDBOX_MEMORY_LIMIT_ENABLED` and `SANDBOX_MEMORY_TOTAL_RESERVE`. diff --git a/deploy/README.md b/deploy/README.md index 9364c75d..df726e07 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -20,7 +20,7 @@ application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-applicati ## Aggregate sandbox memory budget Standalone and the direct Helm chart enable the cgroup v2 aggregate sandbox -memory policy by default: +memory policy by default for the ordinary KVM backend profile: ```toml [plugin.resource.memory_limit] @@ -57,8 +57,18 @@ Helm uses `node.config.sandboxd.config` (`core.node.config.sandboxd.config` in t umbrella chart). Cloud profiles expose the enabled flag and reserve described below, alongside their existing complete-config override. Values use integer bytes or `B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not -accepted. Standard and PVM CI fixtures explicitly use a 2 GiB reserve while -keeping the policy enabled. +accepted. Standard CI explicitly uses a 2 GiB reserve while keeping the policy +enabled. + +The PVM profile keeps this policy disabled by default because enabled parent +enforcement has not been qualified on its matching host. Selecting PVM overrides +the TOML `enabled` value through standalone +`AKERNEL_PVM_MEMORY_LIMIT_ENABLED=false` or Helm +`node.config.sandboxd.pvmMemoryLimitEnabled=false` (`core.` prefix in the umbrella +chart). Explicitly set that switch to `true` and choose a suitable positive +`total_reserve` in the source TOML or complete Helm config to opt in for host +qualification. Both switches accept only `true` or `false` and affect only PVM. +PVM CI retains the disabled policy and does not claim parent-limit coverage. Both cloud Terraform profiles keep the policy disabled by default. Their `sandbox_memory_limit_enabled` input is an explicit opt-in, with diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index fe561416..3f8a5684 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -57,10 +57,21 @@ data: {{- if not (has $firecrackerBackend (list "kvm" "pvm")) }} {{- fail "node.config.sandboxd.firecrackerBackend must be kvm or pvm" }} {{- end }} +{{- $pvmMemoryLimitEnabled := toString .Values.node.config.sandboxd.pvmMemoryLimitEnabled }} +{{- if not (has $pvmMemoryLimitEnabled (list "true" "false")) }} +{{- fail "node.config.sandboxd.pvmMemoryLimitEnabled must be true or false" }} +{{- end }} {{- if eq $firecrackerBackend "pvm" }} {{- if not (and (contains "[plugin.runtime.firecracker]" $sandboxdConfig) (contains "firecracker=\"/usr/local/bin/firecracker\"" $sandboxdConfig)) }} {{- fail "PVM selection requires the standard Firecracker config section and binary entry" }} {{- end }} +{{- $memorySection := regexFind "(?ms)^[\\t ]*\\[plugin\\.resource\\.memory_limit\\][^\\n]*\\n.*?(?:\\n[\\t ]*\\[|\\z)" $sandboxdConfig }} +{{- $memoryEnabledPattern := "(?m)^[\\t ]*enabled[\\t ]*=[\\t ]*(true|false)[\\t ]*(#[^\\n]*)?$" }} +{{- if ne (len (regexFindAll $memoryEnabledPattern $memorySection -1)) 1 }} +{{- fail "PVM selection requires exactly one boolean enabled field under [plugin.resource.memory_limit]" }} +{{- end }} +{{- $pvmMemorySection := regexReplaceAll $memoryEnabledPattern $memorySection (printf "enabled = %s" $pvmMemoryLimitEnabled) }} +{{- $sandboxdConfig = replace $memorySection $pvmMemorySection $sandboxdConfig }} {{- $sandboxdConfig = regexReplaceAll "(?m)^([\\t ]*)firecracker=" $sandboxdConfig "${1}firecracker-pvm=" }} {{- $sandboxdConfig = replace "[plugin.runtime.firecracker]" "[plugin.runtime.firecracker_pvm]" $sandboxdConfig }} {{- $sandboxdConfig = regexReplaceAll "(?m)^[\\t ]*kata=.*$" $sandboxdConfig "# Kata is disabled on the PVM node profile." }} diff --git a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py index 4dffb59d..6c9a9740 100644 --- a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py +++ b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py @@ -73,6 +73,51 @@ def test_complete_config_override_preserves_disabled_and_small_node_policy(self) self.assertEqual(rendered.rstrip(), custom.rstrip()) self.assert_memory_config(tomllib.loads(rendered), enabled, reserve) + def test_pvm_parent_limit_is_disabled_unless_explicitly_enabled(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "pvm-sandboxd.toml" + path.write_text(default.replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"', 1)) + for enabled in ("false", "true"): + with self.subTest(enabled=enabled): + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, enabled == "true", "2GiB") + self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) + resources = render("node.config.sandboxd.firecrackerBackend=pvm") + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, False, "16GiB") + + def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "kvm-sandboxd.toml" + path.write_text(default.replace("enabled = true", "enabled = false", 1)) + resources = render( + "node.config.sandboxd.pvmMemoryLimitEnabled=true", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, False, "16GiB") + + def test_pvm_rejects_invalid_memory_opt_in_and_missing_memory_section(self): + for value in ("1", "TRUE", "yes", ""): + with self.subTest(value=value): + with self.assertRaises(subprocess.CalledProcessError) as error: + render(f"node.config.sandboxd.pvmMemoryLimitEnabled={value}") + self.assertIn("pvmMemoryLimitEnabled must be true or false", error.exception.stderr) + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + path.write_text(default.replace("[plugin.resource.memory_limit]", "[plugin.resource.other]", 1)) + with self.assertRaises(subprocess.CalledProcessError) as error: + render("node.config.sandboxd.firecrackerBackend=pvm", set_files={"node.config.sandboxd.config": path}) + self.assertIn("exactly one boolean enabled field", error.exception.stderr) + def test_control_service_budgets_protect_voluntary_disruption(self): for replicas in (1, 3): resources = render(f"adx.ingressApi.replicas={replicas}") diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 8972851c..d0931c06 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -505,6 +505,9 @@ node: # pvm selects firecracker-pvm and removes Kata from this node profile. # The image must contain the validated PVM guest bundle. firecrackerBackend: kvm + # PVM parent memory enforcement requires separate host qualification. + # Explicit opt-in overrides memory_limit.enabled; set an appropriate reserve in config. + pvmMemoryLimitEnabled: false # Empty uses the distill-fs default (100 GiB on Linux). chunkDbSize: "" config: |- diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index 61e17709..9e04e4fd 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -230,11 +230,14 @@ override themselves. The guest-agent startup budget remains 15 seconds; extending an RPC cleanup budget does not authorize increasing that deadline. The aggregate sandbox memory policy remains disabled by default for PVM until -its finite-cap profile is qualified on the matching maintained host. Ordinary -runsc tests and static PVM profile checks do not qualify this combination. -Explicitly enabling it requires a reserve sized for the selected L1 and a -separate bounded PVM qualification; the 16 GiB deployment reserve is not a -universal host OOM safety budget. +its finite-cap profile is qualified on the matching maintained host. Standalone +can explicitly select `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true`; Helm uses +`node.config.sandboxd.pvmMemoryLimitEnabled=true` (under `core` in the umbrella +chart). Both default to false and apply only to the PVM backend. Select a +`total_reserve` suitable for the actual L1 in its sandboxd configuration before +opting in, then run the matching bounded PVM qualification. Ordinary runsc tests +and static profile checks do not qualify this combination, and the 16 GiB +deployment reserve is not a universal host OOM safety budget. ## Isolated qualification workflow diff --git a/deploy/pvm/run-adx-ci.sh b/deploy/pvm/run-adx-ci.sh index 10c7f6c4..04433dd7 100644 --- a/deploy/pvm/run-adx-ci.sh +++ b/deploy/pvm/run-adx-ci.sh @@ -64,6 +64,9 @@ for name,section,key in [('firecracker','vmm','binary_sha256'),('vmlinux','guest PAYLOAD docker image inspect --format '{{.Id}}' "$IMAGE" > "${out}/image-id.txt" export AKERNEL_FIRECRACKER_BACKEND=pvm AKERNEL_CHUNK_DB_SIZE=1GiB +# Aggregate parent enforcement remains outside the qualified PVM profile. +export AKERNEL_PVM_MEMORY_LIMIT_ENABLED=false +unset AKERNEL_TEST_EXPECT_MEMORY_BYTES AKERNEL_TEST_EXPECT_CPU_MILLIS export AKERNEL_CONTAINER_PROXY=http://10.0.2.2:3128 export AKERNEL_CONTAINER_NO_PROXY=localhost,127.0.0.1,172.17.0.1,10.88.0.1 # Keep Redis AOF/control writes on the L1 root disk while snapshot writes use XFS. @@ -119,10 +122,6 @@ assert text.count(old)==1 text=text.replace(old,'resolv_conf_path="/home/akernel/sandboxd/config/pvm-ci-resolv.conf"') text=text.replace('filestore_dir_size="50G"','filestore_dir_size=""') text=text.replace('filestore_xfs_enabled=false','filestore_xfs_enabled=true') -# The dedicated qualification L1 has 8 GiB; keep the parent policy enabled -# with an explicit test reserve instead of the 16 GiB deployment default. -assert text.count('total_reserve = "16GiB"')==1 -text=text.replace('total_reserve = "16GiB"','total_reserve = "2GiB"',1) # OCI and Nydus directory roots require an explicitly selected virtio-fs # profile; the maintained standalone default remains EROFS. assert text.count('virtiofs_enabled=false')==1 @@ -134,12 +133,6 @@ RESOLVER python3 deploy/pvm/dns-fixture.py > "${out}/dns-fixture.log" 2>&1 & printf '%s\n' "$!" > "${out}/dns-fixture.pid" bash deploy/standalone/start.sh > "${out}/standalone-start.log" 2>&1 -AKERNEL_TEST_EXPECT_MEMORY_BYTES=$(docker exec akernel-node cat /sys/fs/cgroup/akernel/memory.max) -[[ "${AKERNEL_TEST_EXPECT_MEMORY_BYTES}" =~ ^[1-9][0-9]*$ ]] || { - echo 'PVM qualification requires a finite positive sandbox parent memory limit' >&2 - exit 1 -} -export AKERNEL_TEST_EXPECT_MEMORY_BYTES export AKERNEL_TEST_NETWORK_BLOCKED_HOST=blocked.pvm-ci.test export AKERNEL_TEST_NETWORK_ALLOWED_HOST=allowed.pvm-ci.test export AKERNEL_TEST_NETWORK_WILDCARD_HOST=child.blocked.pvm-ci.test @@ -154,7 +147,7 @@ export AKERNEL_TEST_CHECKPOINT_MEMORY_MIB=2048 python3 - "$out" <<'SETTINGS' import json,sys from pathlib import Path -(Path(sys.argv[1])/'qualification-settings.json').write_text(json.dumps({'runtime':'firecracker-pvm','virtiofs_enabled':True,'checkpoint_memory_mib':2048,'other_sdk_memory_mib':2048,'checkpoint_snapshot_type':'Full (requested explicitly by ADX)','debugctl_applied':False,'dedicated_pvm_profile':True},indent=2)) +(Path(sys.argv[1])/'qualification-settings.json').write_text(json.dumps({'runtime':'firecracker-pvm','virtiofs_enabled':True,'checkpoint_memory_mib':2048,'other_sdk_memory_mib':2048,'checkpoint_snapshot_type':'Full (requested explicitly by ADX)','debugctl_applied':False,'dedicated_pvm_profile':True,'sandbox_parent_memory_limit_enabled':False,'sandbox_parent_memory_limit_qualified':False},indent=2)) SETTINGS export PYTHONPATH="${root}/sdk/python" export AKERNEL_TEST_IMAGE=$(python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))["oci"])' "$fixtures") diff --git a/deploy/pvm/test_host_profile.py b/deploy/pvm/test_host_profile.py index f7e8587f..fd9ac3fa 100644 --- a/deploy/pvm/test_host_profile.py +++ b/deploy/pvm/test_host_profile.py @@ -5,6 +5,7 @@ import os from pathlib import Path import subprocess +import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] @@ -46,45 +47,30 @@ def test_shell_exports_are_derived_from_the_json_profile(self): self.assertEqual(result.stdout.splitlines(), list(expected.values())) -class CiMemoryBudgetTest(unittest.TestCase): - def measured_budget(self, value, docker_status=0): +class CiMemoryPolicyTest(unittest.TestCase): + def test_pvm_ci_keeps_parent_enforcement_disabled_even_if_inherited_enabled(self): script = (ROOT / "deploy/pvm/run-adx-ci.sh").read_text() - after_start = script.split( - 'bash deploy/standalone/start.sh > "${out}/standalone-start.log" 2>&1\n', 1 - )[1] - preparation = after_start.split("export AKERNEL_TEST_NETWORK_BLOCKED_HOST=", 1)[0] - environment = dict(os.environ) - environment.pop("AKERNEL_TEST_EXPECT_MEMORY_BYTES", None) - environment.update(MEASURED_MEMORY_BYTES=value, DOCKER_STATUS=str(docker_status)) - return subprocess.run( - ["bash", "-euc", ''' -docker() { - [[ "$*" == "exec akernel-node cat /sys/fs/cgroup/akernel/memory.max" ]] || return 99 - printf '%s\\n' "${MEASURED_MEMORY_BYTES}" - return "${DOCKER_STATUS}" -} -''' + preparation + ''' -python3 -c 'import os; print(os.environ["AKERNEL_TEST_EXPECT_MEMORY_BYTES"])' -'''], - text=True, capture_output=True, env=environment, timeout=10, + preparation = "export AKERNEL_FIRECRACKER_BACKEND=pvm" + script.split( + "export AKERNEL_FIRECRACKER_BACKEND=pvm", 1 + )[1].split("export AKERNEL_CONTAINER_PROXY=", 1)[0] + result = subprocess.run( + ["bash", "-euc", preparation + '\npython3 -c \'import os; print(os.environ["AKERNEL_PVM_MEMORY_LIMIT_ENABLED"]); assert "AKERNEL_TEST_EXPECT_MEMORY_BYTES" not in os.environ; assert "AKERNEL_TEST_EXPECT_CPU_MILLIS" not in os.environ\''], + text=True, capture_output=True, + env=dict(os.environ, AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true", + AKERNEL_TEST_EXPECT_MEMORY_BYTES="6442450944", + AKERNEL_TEST_EXPECT_CPU_MILLIS="1000"), timeout=10, ) - - def test_pvm_sdk_tests_inherit_the_measured_parent_limit(self): - maximum = str(6 * 1024**3 + 4096) - result = self.measured_budget(maximum) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), maximum) - - def test_pvm_qualification_rejects_missing_or_unlimited_parent_caps(self): - for value in ("max", "0", "-1", ""): - with self.subTest(value=value): - result = self.measured_budget(value) - self.assertNotEqual(result.returncode, 0) - self.assertIn("finite positive sandbox parent memory limit", result.stderr) + self.assertEqual(result.stdout.strip(), "false") - def test_pvm_qualification_preserves_parent_probe_failures(self): - result = self.measured_budget(str(6 * 1024**3), docker_status=17) - self.assertEqual(result.returncode, 17) + def test_pvm_qualification_settings_do_not_claim_parent_limit_coverage(self): + script = (ROOT / "deploy/pvm/run-adx-ci.sh").read_text() + settings = script.split("<<'SETTINGS'\n", 1)[1].split("\nSETTINGS", 1)[0] + with tempfile.TemporaryDirectory() as directory: + subprocess.run(["python3", "-c", settings, directory], check=True, timeout=10) + recorded = json.loads((Path(directory) / "qualification-settings.json").read_text()) + self.assertFalse(recorded["sandbox_parent_memory_limit_enabled"]) + self.assertFalse(recorded["sandbox_parent_memory_limit_qualified"]) if __name__ == "__main__": diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 0606d32f..ed2981ac 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -58,7 +58,8 @@ for inspection and retry. Data and credentials survive stopping. For an independent data profile, set an absolute `AKERNEL_STANDALONE_DATA_DIR` for every command. Only one standalone node can run per Docker daemon. -Standalone enables the aggregate sandbox memory limit with a 16 GiB reserve. +The ordinary KVM backend profile enables the aggregate sandbox memory limit +with a 16 GiB reserve. It requires cgroup v2 and effective memory above that reserve, including any Docker/OrbStack VM or visible ancestor limit. Before starting on a smaller machine, edit `[plugin.resource.memory_limit]` in @@ -69,6 +70,14 @@ applies to `make standalone-source-test`. See [the memory budget and upgrade contract](../README.md#aggregate-sandbox-memory-budget) before applying the new default to an existing profile. +Selecting `AKERNEL_FIRECRACKER_BACKEND=pvm` keeps the parent limit disabled by +default, including when the source TOML has `enabled=true`. PVM parent-limit +enforcement still needs qualification on the matching host. To opt in for that +qualification, set `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` and choose a suitable +positive `total_reserve` in the source TOML before startup. The switch accepts +only `true` or `false`, defaults to `false`, and affects only PVM. The maintained +PVM CI fixture keeps it disabled. + The bundled ADX configuration takes scheduling capacity from sandboxd's `/var/run/resource.sock`, including PVM and trace/shared-storage variants. The image build resolves the default profile before replacing the entire diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index 7dfe1b8e..a933d1d3 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -24,6 +24,7 @@ AKERNEL_DATA_PORT="${AKERNEL_DATA_PORT:-80}" AKERNEL_ENDPOINT_HOST="${AKERNEL_ENDPOINT_HOST:-127.0.0.1}" AKERNEL_CHUNK_DB_SIZE="${AKERNEL_CHUNK_DB_SIZE:-}" AKERNEL_FIRECRACKER_BACKEND="${AKERNEL_FIRECRACKER_BACKEND:-kvm}" +AKERNEL_PVM_MEMORY_LIMIT_ENABLED="${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" # Container runtime command (docker or pouch) DOCKER_CMD="" @@ -191,6 +192,13 @@ check_prerequisites() { exit 1 ;; esac + case "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED}" in + true|false) ;; + *) + log_error "AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false" + exit 1 + ;; + esac if [[ "${DATA_DIR}" != /* ]]; then log_error "AKERNEL_STANDALONE_DATA_DIR must be an absolute path" exit 1 @@ -323,6 +331,13 @@ configure_network() { ) if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]]; then + case "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" in + true|false) ;; + *) + log_error "AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false" + exit 1 + ;; + esac if ! grep -q '^\[plugin.runtime.firecracker\]$' \ "${CONFIG_DIR}/sandboxd_config.toml" || \ ! grep -q '^firecracker="/usr/local/bin/firecracker"$' \ @@ -330,10 +345,21 @@ configure_network() { log_error "PVM selection requires the standard Firecracker config section and binary entry" exit 1 fi + if ! awk ' + /^[[:space:]]*\[/ { + in_memory = ($0 ~ /^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/) + } + in_memory && /^[[:space:]]*enabled[[:space:]]*=[[:space:]]*(true|false)[[:space:]]*(#.*)?$/ { count++ } + END { exit !(count == 1) } + ' "${CONFIG_DIR}/sandboxd_config.toml"; then + log_error "PVM selection requires exactly one boolean enabled field under [plugin.resource.memory_limit]" + exit 1 + fi sed_args+=( -e 's/^firecracker=/firecracker-pvm=/' -e 's/^\[plugin.runtime.firecracker\]$/[plugin.runtime.firecracker_pvm]/' -e '/^kata=/d' + -e "/^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/,/^[[:space:]]*\[/s/^([[:space:]]*)enabled[[:space:]]*=.*/\1enabled = ${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}/" ) fi diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index 31c38904..636c5471 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -57,6 +57,7 @@ def generate(self, **overrides): AKERNEL_NAT_BACKEND="iptables", AKERNEL_ENABLE_RUNC="true", AKERNEL_FIRECRACKER_BACKEND="kvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="false", AKERNEL_RUNC_RESOLV_CONF=str(self.source), AKERNEL_CHUNK_DB_SIZE="", ) @@ -137,6 +138,42 @@ def test_pvm_selection_preserves_direct_resolver_configuration(self): self.assertNotIn("firecracker", config["plugin.runtime.runtime_binary"]) self.assertNotIn("kata", config["plugin.runtime.runtime_binary"]) self.assertEqual(self.resolver_output.read_text(), self.resolver_content) + self.assert_memory_config("false", "16GiB") + + def test_pvm_parent_limit_requires_explicit_opt_in_and_preserves_reserve(self): + self.template.write_text( + self.template.read_text().replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"') + ) + for enabled in ("false", "true"): + with self.subTest(enabled=enabled): + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, "2GiB") + + def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): + self.template.write_text(self.template.read_text().replace("enabled = true", "enabled = false", 1)) + result = self.generate(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config("false", "16GiB") + + def test_pvm_rejects_invalid_memory_opt_in_and_missing_memory_section(self): + for value in ("", "1", "TRUE", "yes", "true; false"): + with self.subTest(value=value): + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=value, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false", result.stderr) + self.assertFalse(self.config_output.exists()) + self.template.write_text(self.template.read_text().replace("[plugin.resource.memory_limit]", "[plugin.resource.other]", 1)) + result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") + self.assertNotEqual(result.returncode, 0) + self.assertIn("exactly one boolean enabled field", result.stderr) + self.assertFalse(self.config_output.exists()) def test_direct_source_is_under_runtime_and_node_source_is_preserved(self): for backend in ("iptables", "bpfnat"): diff --git a/deploy/standalone/test_start_platform.py b/deploy/standalone/test_start_platform.py index 3126eb4c..683f0d16 100644 --- a/deploy/standalone/test_start_platform.py +++ b/deploy/standalone/test_start_platform.py @@ -57,6 +57,7 @@ def setUp(self): YR_IMAGE_PROCESS_CONFIG="/run/akernel/yr-image-process.json", AKERNEL_ENABLE_RUNC="true", AKERNEL_NAT_BACKEND="iptables", AKERNEL_FIRECRACKER_BACKEND="kvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="false", MOCK_HOST="Darwin", MOCK_OS="OrbStack", MOCK_ENGINE="linux/aarch64", MOCK_IMAGE="linux/arm64", MOCK_TRAEFIK_IMAGE="linux/arm64", @@ -176,6 +177,12 @@ def test_invalid_firecracker_backend_is_rejected_before_profile_mutation(self): self.assertIn("AKERNEL_FIRECRACKER_BACKEND must be kvm or pvm", result.stderr) self.assert_no_profile_mutation() + def test_invalid_pvm_memory_opt_in_is_rejected_before_profile_mutation(self): + result = self.run_main(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="yes") + self.assertNotEqual(result.returncode, 0) + self.assertIn("AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false", result.stderr) + self.assert_no_profile_mutation() + def test_native_runc_uses_only_disposable_capability_probe(self): result = self.run_script( "validate_host_platform\nvalidate_image_capabilities\nprepare_host_network_modules" diff --git a/sdk/python/tests/e2e/README.md b/sdk/python/tests/e2e/README.md index 366743aa..d8906261 100644 --- a/sdk/python/tests/e2e/README.md +++ b/sdk/python/tests/e2e/README.md @@ -17,6 +17,14 @@ A public SDK Full run is not the entire platform deployment acceptance suite. Most cases request 1 CPU / 2 GiB per Sandbox; concurrent fixtures need capacity for multiple instances. +The `sdk-contracts` group runs `SandboxPublicContractIntegrationTest` for every +runtime. Parent memory-budget qualification is a separate +`MemoryBudgetIntegrationTest` in the same module, discovered by standard CI's +standalone unittest command with `AKERNEL_TEST_EXPECT_MEMORY_BYTES` set to the +measured finite parent limit. It also checks CPU when +`AKERNEL_TEST_EXPECT_CPU_MILLIS` is set. The PVM fixture leaves parent enforcement +disabled and does not include this qualification in its SDK smoke coverage. + ## Run Install the SDK from this checkout and configure its diff --git a/sdk/python/tests/e2e/run.py b/sdk/python/tests/e2e/run.py index c4d6cbf6..e4960467 100644 --- a/sdk/python/tests/e2e/run.py +++ b/sdk/python/tests/e2e/run.py @@ -31,7 +31,7 @@ class Case: Case( "sdk-contracts", "l0", - ("-m", "unittest", "-v", "tests.e2e.standalone.test_sdk_contracts"), + ("-m", "unittest", "-v", "tests.e2e.standalone.test_sdk_contracts.SandboxPublicContractIntegrationTest"), 300, ), Case( diff --git a/src/sandboxd b/src/sandboxd index fbe69e56..a4ba0cf6 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit fbe69e566859718414a207fa780b8dab363b9381 +Subproject commit a4ba0cf6aedef1431371b8a66034963cf8685b4b From 629dc4d966486ca96e55e97a82d3825a8ffd5788 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 17:07:48 +0800 Subject: [PATCH 7/9] fix(deploy): preserve optional PVM memory configuration Keep older custom PVM profiles valid when the disabled memory policy is omitted. Normalize supported quoted boolean keys while retaining strict validation for duplicate fields, invalid types and explicit opt-in. Signed-off-by: wanghongtao.wangho --- .../charts/core/templates/node/configmap.yaml | 13 ++- .../tests/test_shared_checkpoint_storage.py | 85 ++++++++++++++----- deploy/standalone/start.sh | 12 +-- deploy/standalone/test_start_config.py | 74 ++++++++++++---- 4 files changed, 139 insertions(+), 45 deletions(-) diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index 3f8a5684..525fc4ff 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -66,12 +66,17 @@ data: {{- fail "PVM selection requires the standard Firecracker config section and binary entry" }} {{- end }} {{- $memorySection := regexFind "(?ms)^[\\t ]*\\[plugin\\.resource\\.memory_limit\\][^\\n]*\\n.*?(?:\\n[\\t ]*\\[|\\z)" $sandboxdConfig }} -{{- $memoryEnabledPattern := "(?m)^[\\t ]*enabled[\\t ]*=[\\t ]*(true|false)[\\t ]*(#[^\\n]*)?$" }} -{{- if ne (len (regexFindAll $memoryEnabledPattern $memorySection -1)) 1 }} -{{- fail "PVM selection requires exactly one boolean enabled field under [plugin.resource.memory_limit]" }} -{{- end }} +{{- $memoryEnabledPattern := "(?m)^[\\t ]*(enabled|\"enabled\"|'enabled')[\\t ]*=[\\t ]*(true|false)[\\t ]*(#[^\\n]*)?$" }} +{{- $memoryEnabledCount := len (regexFindAll $memoryEnabledPattern $memorySection -1) }} +{{- $memoryEnabledFields := len (regexFindAll "(?m)^[\\t ]*(enabled|\"enabled\"|'enabled')[\\t ]*=" $memorySection -1) }} +{{- if or (ne $memoryEnabledFields $memoryEnabledCount) (gt $memoryEnabledCount 1) (and (eq $pvmMemoryLimitEnabled "true") (ne $memoryEnabledCount 1)) }} +{{- fail "PVM selection requires at most one boolean enabled field under [plugin.resource.memory_limit], and exactly one when opting in" }} +{{- end }} +{{- /* Older custom profiles may omit the optional, disabled memory policy. */}} +{{- if eq $memoryEnabledCount 1 }} {{- $pvmMemorySection := regexReplaceAll $memoryEnabledPattern $memorySection (printf "enabled = %s" $pvmMemoryLimitEnabled) }} {{- $sandboxdConfig = replace $memorySection $pvmMemorySection $sandboxdConfig }} +{{- end }} {{- $sandboxdConfig = regexReplaceAll "(?m)^([\\t ]*)firecracker=" $sandboxdConfig "${1}firecracker-pvm=" }} {{- $sandboxdConfig = replace "[plugin.runtime.firecracker]" "[plugin.runtime.firecracker_pvm]" $sandboxdConfig }} {{- $sandboxdConfig = regexReplaceAll "(?m)^[\\t ]*kata=.*$" $sandboxdConfig "# Kata is disabled on the PVM node profile." }} diff --git a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py index 6c9a9740..1ed6c6bd 100644 --- a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py +++ b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py @@ -4,6 +4,7 @@ from __future__ import annotations +import re import subprocess import tempfile import tomllib @@ -77,17 +78,20 @@ def test_pvm_parent_limit_is_disabled_unless_explicitly_enabled(self): default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "pvm-sandboxd.toml" - path.write_text(default.replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"', 1)) - for enabled in ("false", "true"): - with self.subTest(enabled=enabled): - resources = render( - "node.config.sandboxd.firecrackerBackend=pvm", - f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", - set_files={"node.config.sandboxd.config": path}, - ) - config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) - self.assert_memory_config(config, enabled == "true", "2GiB") - self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) + original = default.replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"', 1) + for key in ("enabled", '"enabled"', "'enabled'"): + for source_enabled in ("false", "true"): + path.write_text(original.replace("enabled = true", f"{key} = {source_enabled}", 1)) + for enabled in ("false", "true"): + with self.subTest(key=key, source_enabled=source_enabled, enabled=enabled): + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, enabled == "true", "2GiB") + self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) resources = render("node.config.sandboxd.firecrackerBackend=pvm") config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) self.assert_memory_config(config, False, "16GiB") @@ -104,19 +108,62 @@ def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) self.assert_memory_config(config, False, "16GiB") - def test_pvm_rejects_invalid_memory_opt_in_and_missing_memory_section(self): + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + without_table = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "", default, + ) + without_enabled = default.replace("enabled = true\n", "", 1) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for custom, has_table in ((without_table, False), (without_enabled, True)): + with self.subTest(has_table=has_table): + path.write_text(custom) + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + memory = config["plugin"]["resource"].get("memory_limit", {}) + self.assertFalse(memory.get("enabled", False)) + self.assertEqual("memory_limit" in config["plugin"]["resource"], has_table) + if has_table: + self.assertEqual(memory, {"total_reserve": "16GiB"}) + self.assertIn("firecracker_pvm", config["plugin"]["runtime"]) + self.assertIn("firecracker-pvm", config["plugin"]["runtime"]["runtime_binary"]) + with self.assertRaises(subprocess.CalledProcessError) as error: + render( + "node.config.sandboxd.firecrackerBackend=pvm", + "node.config.sandboxd.pvmMemoryLimitEnabled=true", + set_files={"node.config.sandboxd.config": path}, + ) + self.assertIn("exactly one when opting in", error.exception.stderr) + + def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "custom-sandboxd.toml" + for field in ('enabled = "true"', "enabled = 1", '"enabled" = "true"', "'enabled' = 1", + "enabled = true\nenabled = false", 'enabled = true\n"enabled" = false', + "enabled = true\n'enabled' = false"): + for opt_in in ("false", "true"): + with self.subTest(field=field, opt_in=opt_in): + path.write_text(default.replace("enabled = true", field, 1)) + with self.assertRaises(subprocess.CalledProcessError) as error: + render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={opt_in}", + set_files={"node.config.sandboxd.config": path}, + ) + self.assertIn("boolean enabled field", error.exception.stderr) + + def test_pvm_rejects_invalid_memory_opt_in(self): for value in ("1", "TRUE", "yes", ""): with self.subTest(value=value): with self.assertRaises(subprocess.CalledProcessError) as error: render(f"node.config.sandboxd.pvmMemoryLimitEnabled={value}") self.assertIn("pvmMemoryLimitEnabled must be true or false", error.exception.stderr) - default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "custom-sandboxd.toml" - path.write_text(default.replace("[plugin.resource.memory_limit]", "[plugin.resource.other]", 1)) - with self.assertRaises(subprocess.CalledProcessError) as error: - render("node.config.sandboxd.firecrackerBackend=pvm", set_files={"node.config.sandboxd.config": path}) - self.assertIn("exactly one boolean enabled field", error.exception.stderr) def test_control_service_budgets_protect_voluntary_disruption(self): for replicas in (1, 3): diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index a933d1d3..f3ab9a98 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -345,21 +345,23 @@ configure_network() { log_error "PVM selection requires the standard Firecracker config section and binary entry" exit 1 fi - if ! awk ' + # Older custom profiles may omit the optional, disabled memory policy. + if ! awk -v opt_in="${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" ' /^[[:space:]]*\[/ { in_memory = ($0 ~ /^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/) } - in_memory && /^[[:space:]]*enabled[[:space:]]*=[[:space:]]*(true|false)[[:space:]]*(#.*)?$/ { count++ } - END { exit !(count == 1) } + in_memory && /^[[:space:]]*(enabled|"enabled"|\047enabled\047)[[:space:]]*=/ { fields++ } + in_memory && /^[[:space:]]*(enabled|"enabled"|\047enabled\047)[[:space:]]*=[[:space:]]*(true|false)[[:space:]]*(#.*)?$/ { booleans++ } + END { exit !(fields == booleans && booleans <= 1 && (opt_in == "false" || booleans == 1)) } ' "${CONFIG_DIR}/sandboxd_config.toml"; then - log_error "PVM selection requires exactly one boolean enabled field under [plugin.resource.memory_limit]" + log_error "PVM selection requires at most one boolean enabled field under [plugin.resource.memory_limit], and exactly one when opting in" exit 1 fi sed_args+=( -e 's/^firecracker=/firecracker-pvm=/' -e 's/^\[plugin.runtime.firecracker\]$/[plugin.runtime.firecracker_pvm]/' -e '/^kata=/d' - -e "/^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/,/^[[:space:]]*\[/s/^([[:space:]]*)enabled[[:space:]]*=.*/\1enabled = ${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}/" + -e "/^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/,/^[[:space:]]*\[/s/^([[:space:]]*)(enabled|\"enabled\"|'enabled')[[:space:]]*=.*/\1enabled = ${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}/" ) fi diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index 636c5471..6d2f6c08 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -141,17 +141,18 @@ def test_pvm_selection_preserves_direct_resolver_configuration(self): self.assert_memory_config("false", "16GiB") def test_pvm_parent_limit_requires_explicit_opt_in_and_preserves_reserve(self): - self.template.write_text( - self.template.read_text().replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"') - ) - for enabled in ("false", "true"): - with self.subTest(enabled=enabled): - result = self.generate( - AKERNEL_FIRECRACKER_BACKEND="pvm", - AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assert_memory_config(enabled, "2GiB") + original = self.template.read_text().replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"') + for key in ("enabled", '"enabled"', "'enabled'"): + for source_enabled in ("false", "true"): + self.template.write_text(original.replace("enabled = true", f"{key} = {source_enabled}", 1)) + for enabled in ("false", "true"): + with self.subTest(key=key, source_enabled=source_enabled, enabled=enabled): + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, "2GiB") def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): self.template.write_text(self.template.read_text().replace("enabled = true", "enabled = false", 1)) @@ -159,7 +160,51 @@ def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): self.assertEqual(result.returncode, 0, result.stderr) self.assert_memory_config("false", "16GiB") - def test_pvm_rejects_invalid_memory_opt_in_and_missing_memory_section(self): + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): + original = self.template.read_text() + without_table = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + "", original, + ) + without_enabled = original.replace("enabled = true\n", "", 1) + for custom, has_table in ((without_table, False), (without_enabled, True)): + with self.subTest(has_table=has_table): + self.template.write_text(custom) + result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") + self.assertEqual(result.returncode, 0, result.stderr) + config = self.read_sections() + self.assertFalse(config.getboolean("plugin.resource.memory_limit", "enabled", fallback=False)) + self.assertEqual(config.has_section("plugin.resource.memory_limit"), has_table) + if has_table: + self.assertEqual(dict(config["plugin.resource.memory_limit"]), {"total_reserve": '"16GiB"'}) + self.assertIn("plugin.runtime.firecracker_pvm", config) + self.assertIn("firecracker-pvm", config["plugin.runtime.runtime_binary"]) + generated = self.config_output.read_text() + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true", + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("exactly one when opting in", result.stderr) + self.assertEqual(self.config_output.read_text(), generated) + + def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): + original = self.template.read_text() + for field in ('enabled = "true"', "enabled = 1", '"enabled" = "true"', "'enabled' = 1", + "enabled = true\nenabled = false", 'enabled = true\n"enabled" = false', + "enabled = true\n'enabled' = false"): + for opt_in in ("false", "true"): + with self.subTest(field=field, opt_in=opt_in): + self.template.write_text(original.replace("enabled = true", field, 1)) + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=opt_in, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("boolean enabled field", result.stderr) + self.assertFalse(self.config_output.exists()) + + def test_pvm_rejects_invalid_memory_opt_in(self): for value in ("", "1", "TRUE", "yes", "true; false"): with self.subTest(value=value): result = self.generate( @@ -169,11 +214,6 @@ def test_pvm_rejects_invalid_memory_opt_in_and_missing_memory_section(self): self.assertNotEqual(result.returncode, 0) self.assertIn("AKERNEL_PVM_MEMORY_LIMIT_ENABLED must be true or false", result.stderr) self.assertFalse(self.config_output.exists()) - self.template.write_text(self.template.read_text().replace("[plugin.resource.memory_limit]", "[plugin.resource.other]", 1)) - result = self.generate(AKERNEL_FIRECRACKER_BACKEND="pvm") - self.assertNotEqual(result.returncode, 0) - self.assertIn("exactly one boolean enabled field", result.stderr) - self.assertFalse(self.config_output.exists()) def test_direct_source_is_under_runtime_and_node_source_is_preserved(self): for backend in ("iptables", "bpfnat"): From bc3aabdd01a8be7176cbcd6e89f56827b707cef7 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Fri, 6 Nov 2026 17:36:36 +0800 Subject: [PATCH 8/9] fix(deploy): narrow default memory enforcement to Helm Leave standalone memory enforcement disabled and retain its existing auto capacity source. Defer the source switch and remove the independent CPU candidate from the parent dependency pins. Keep cloud and PVM profiles explicitly opt-in. Restore the ordinary standalone CI and SDK contracts, and verify with the packaged CLI that all maintained standalone variants preserve their original auto source. Signed-off-by: wanghongtao.wangho --- .github/workflows/ci.yml | 40 ----- AGENTS.md | 48 +++--- builder/node.Dockerfile | 10 +- builder/scripts/prepare-adx-config.py | 54 +------ builder/scripts/test_afs_config.py | 139 +++--------------- deploy/README.md | 50 +++---- deploy/pvm-runtime.md | 20 +-- deploy/standalone/README.md | 34 ++--- deploy/standalone/config/sandboxd_config.toml | 4 +- deploy/standalone/test_start_config.py | 21 +-- sdk/python/tests/e2e/README.md | 8 - sdk/python/tests/e2e/run.py | 2 +- .../e2e/standalone/test_sdk_contracts.py | 71 --------- .../unit/test_e2e_memory_budget_fixture.py | 84 ----------- src/sandboxd | 2 +- 15 files changed, 118 insertions(+), 469 deletions(-) delete mode 100644 sdk/python/tests/unit/test_e2e_memory_budget_fixture.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 126699e7..58ff27a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -267,23 +267,6 @@ jobs: test ! -e /usr/local/bin/runc-shim ' - - name: Select the bounded CI memory profile - run: | - # The runner can have less memory than the deployment's 16 GiB reserve. - # Exercise the enabled policy with an explicit smaller test reserve. - python3 - <<'PY' - from pathlib import Path - import tomllib - path = Path("deploy/standalone/config/sandboxd_config.toml") - text = path.read_text() - assert tomllib.loads(text)["plugin"]["resource"]["memory_limit"] == { - "enabled": True, "total_reserve": "16GiB" - } - old = 'total_reserve = "16GiB"' - assert text.count(old) == 1 - path.write_text(text.replace(old, 'total_reserve = "2GiB"', 1)) - PY - - name: Start standalone AKernel run: | IMAGE="akernel-ci/all-in-one:${GITHUB_SHA}" \ @@ -292,29 +275,6 @@ jobs: AKERNEL_DATA_BIND=127.0.0.1 \ ./deploy/standalone/start.sh - - name: Verify the enabled parent memory policy - run: | - docker exec -i akernel-node python3 - <<'PY' - import json - import socket - from http.client import HTTPConnection - from pathlib import Path - maximum = int(Path("/sys/fs/cgroup/akernel/memory.max").read_text()) - assert maximum > 0 - connection = HTTPConnection("localhost", timeout=10) - connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - connection.sock.settimeout(10) - connection.sock.connect("/var/run/resource.sock") - connection.request("GET", "/resource") - response = connection.getresponse() - assert response.status == 200, response.status - capacity = json.load(response)["mem"] - connection.close() - assert capacity == maximum, (capacity, maximum) - print(f"finite parent memory.max and /resource agree: {maximum} bytes") - PY - echo "AKERNEL_TEST_EXPECT_MEMORY_BYTES=$(docker exec akernel-node cat /sys/fs/cgroup/akernel/memory.max)" >> "${GITHUB_ENV}" - - name: Trust the standalone HTTPS certificate run: | docker cp akernel-node:/home/akernel/adx/tls/ingress-public.pem \ diff --git a/AGENTS.md b/AGENTS.md index ed69f2e4..7fdeb2cd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -240,28 +240,26 @@ distill-fs release's packaged manifest. Use [`deploy/README.md`](./deploy/README.md) as the deployment entry point. AKernel supports standalone, existing Kubernetes clusters via Helm, and Terraform-based cloud provisioning. -Standalone and the direct Helm chart enable the cgroup v2 parent memory policy -with `total_reserve="16GiB"` for the ordinary KVM profile. PVM keeps it disabled -until explicitly selected with `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` in -standalone or `node.config.sandboxd.pvmMemoryLimitEnabled=true` in Helm. Size -the reserve for the selected L1 and qualify that finite-cap combination on the +The direct Helm chart enables the cgroup v2 parent memory policy with +`total_reserve="16GiB"` for the ordinary KVM profile. Standalone and both cloud +Terraform profiles keep it disabled by default. Cloud opt-in uses +`sandbox_memory_limit_enabled`; guided profiles expose +`SANDBOX_MEMORY_LIMIT_ENABLED` and `SANDBOX_MEMORY_TOTAL_RESERVE`. Qualify cgroup +v2 and effective memory (host MemTotal clipped by visible ancestor limits) +above the reserve with room for sandbox requests before enabling the policy. +The bundled standalone ADX profile retains its `auto` resource source, which +does not guarantee scheduling capacity follows the sandbox parent ceiling. +Operators enabling the policy must select and qualify a compatible +`resource_source`, including memory capacity, allocation/release accounting and +schedulable CPU. The disabled default does not fix resource-source limitations. +PVM also keeps the policy disabled until explicitly selected with +`AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true` in standalone or +`node.config.sandboxd.pvmMemoryLimitEnabled=true` in Helm. Size the reserve for +the selected L1 and qualify the finite-cap and scheduling combination on the matching PVM host before rollout; ordinary runsc tests do not qualify PVM. -Both cloud Terraform profiles leave it disabled -until explicitly enabled with `sandbox_memory_limit_enabled`; their default -operating-system images have not been qualified for this policy. Guided profiles -expose `SANDBOX_MEMORY_LIMIT_ENABLED` and `SANDBOX_MEMORY_TOTAL_RESERVE`. -Effective memory (host MemTotal clipped by visible ancestor limits) must exceed -that reserve and leave room for sandbox requests. -Small Docker/OrbStack VMs and CI fixtures must explicitly select a smaller -reserve or disable the policy in their sandboxd configuration. Keep the budget -semantics aligned across standalone, Helm and both cloud templates. -The bundled standalone ADX configuration is resolved at image build time and -uses sandboxd's resource socket; replace the complete tagged capacity source, -preserving explicitly selected custom sources. Public scheduling capacity must -follow the sandbox parent ceiling, with allocatable decreasing and recovering -as requests are allocated and deleted. Adding or lowering a parent cap requires -an empty sandbox subtree; an existing stricter cap is retained. This aggregate -sandbox cap does not bound all node components or guarantee avoidance of host OOM. +Adding or lowering a parent cap requires an empty sandbox subtree; an existing +stricter cap is retained. This aggregate sandbox cap does not bound all node +components or guarantee avoidance of host OOM. See `deploy/README.md#aggregate-sandbox-memory-budget` for migration and scope. The [standalone guide](./deploy/standalone/README.md) owns source startup, supported platforms, runtime selection, and SDK setup. Preserve the source launcher's native-daemon architecture selection, clean sandboxd gitlink check, occupied-name preflight, profile locking, unique source image, and node/gateway health checks. Status and stop must verify recorded container identities, images, and data mounts; stop also requires an empty sandbox inventory. Source management is independent of cloud profiles and must preserve unrelated containers. @@ -758,14 +756,6 @@ entirely. Do not publish per-commit SHA tags or other historical image tags. The job checks the image contains runsc and excludes Kata, Firecracker, virtiofsd, and runc before starting standalone E2E. -The bounded standalone CI profile selects a 2 GiB reserve and exports the -measured parent limit as `AKERNEL_TEST_EXPECT_MEMORY_BYTES`. The SDK E2E -regression checks public capacity plus allocation and release against that -single-node budget; retain the parent/socket check as a separate assertion. -Public memory values round down to whole MiB. Keep page-aligned, non-MiB parent -fixtures and optional `AKERNEL_TEST_EXPECT_CPU_MILLIS` qualification coverage so -fractional CPU budgets can be exercised with schedulable requests. - CI explicitly trusts the generated standalone public certificate through `SSL_CERT_FILE` and enables `AKERNEL_VERIFY_TLS=1` before SDK examples and integration tests. Keep HTTPS certificate and hostname validation enabled. diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index fd023be8..a01bdb1a 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -476,13 +476,11 @@ COPY ./builder/scripts/sandboxd_network_prepare.sh /usr/local/bin/sandboxd-netwo ARG FIRECRACKER_KERNEL_PROFILE COPY ./builder/config/adx-standalone.yaml /etc/akernel/adx-standalone.yaml RUN case "${FIRECRACKER_KERNEL_PROFILE}" in \ - pvm) set -- --pvm ;; \ - kvm) set -- ;; \ + pvm) python3 /usr/local/libexec/prepare-adx-config.py \ + /etc/akernel/adx-standalone.yaml /etc/akernel/adx-standalone.yaml --pvm ;; \ + kvm) ;; \ *) echo "invalid Firecracker kernel profile" >&2; exit 1 ;; \ - esac; \ - python3 /usr/local/libexec/prepare-adx-config.py \ - /etc/akernel/adx-standalone.yaml /etc/akernel/adx-standalone.yaml \ - --sandboxd-capacity "$@" + esac RUN chmod 0755 \ /usr/local/bin/runsc \ /usr/local/bin/sandboxd \ diff --git a/builder/scripts/prepare-adx-config.py b/builder/scripts/prepare-adx-config.py index c114920e..26901917 100644 --- a/builder/scripts/prepare-adx-config.py +++ b/builder/scripts/prepare-adx-config.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare the bundled capacity source, PVM, telemetry and checkpoint options.""" +"""Apply PVM, telemetry and mounted checkpoint options to ADX deployments.""" from __future__ import annotations import argparse import os @@ -10,50 +10,6 @@ import yaml -def standalone_sandboxd_capacity(cfg: dict, source: Path, adxctl: Path) -> dict: - """Resolve the static bundled profile before replacing its tagged source. - - ADX recursively merges profile overrides, which would retain auto's - disk_path when switching to sandboxd. Custom configurations with an - explicit source remain caller-owned, including explicit auto/file sources. - This build-time conversion does not expand custom runtime expressions. - """ - if cfg.get("profile") != "standalone": - return cfg - node = cfg.get("service_overrides", {}).get("adxlet", {}).get("config") or {} - if "capacity_file" in node or node.get("resource_source") != { - "disk_path": "/home/akernel" - }: - return cfg - - def has_expression(value): - if isinstance(value, str): - return "$" in value - if isinstance(value, dict): - return any(has_expression(item) for item in value.values()) - if isinstance(value, list): - return any(has_expression(item) for item in value) - return False - - if has_expression(cfg): - raise ValueError("bundled capacity conversion requires a static deployment") - resolved = yaml.safe_load(subprocess.run( - [str(adxctl), "--config", str(source), "config", "dump"], - check=True, capture_output=True, text=True, - ).stdout) - if not isinstance(resolved, dict) or "profile" in resolved: - raise ValueError("ADX did not resolve the standalone deployment") - workers = [service for service in resolved["services"] - if service["role"] == "adxlet"] - if len(workers) != 1: - raise ValueError("bundled standalone deployment must contain one adxlet") - workers[0]["config"]["resource_source"] = { - "kind": "sandboxd", "socket": "/var/run/resource.sock", - "valid_for_seconds": 30, - } - return resolved - - def expand_selector(value: str) -> str: """Match ADX's single-pass ${NAME:-default}/$$ discriminator expansion. @@ -96,13 +52,10 @@ def expand_selector(value: str) -> str: def configure( source: Path, destination: Path, mount: Path | None, prefix: str, *, pvm: bool = False, - sandboxd_capacity: bool = False, adxctl: Path = Path("/opt/adx/current/bin/adxctl"), ) -> None: cfg = yaml.safe_load(source.read_text()) if not isinstance(cfg, dict): raise ValueError("deployment must be a mapping") - if sandboxd_capacity: - cfg = standalone_sandboxd_capacity(cfg, source, adxctl) profile_roles = { "standalone": ("coordinator", "apiserver", "adxlet", "ingress"), "standalone-external-redis": ("coordinator", "apiserver", "adxlet", "ingress"), @@ -181,11 +134,8 @@ def main() -> None: parser.add_argument("--mount", type=Path, default=None) parser.add_argument("--prefix", default="akernel/checkpoints") parser.add_argument("--pvm", action="store_true", help="use the PVM cleanup RPC budget") - parser.add_argument("--sandboxd-capacity", action="store_true", - help="resolve the static bundled profile and use sandboxd capacity") args = parser.parse_args() - configure(args.source, args.destination, args.mount, args.prefix, pvm=args.pvm, - sandboxd_capacity=args.sandboxd_capacity) + configure(args.source, args.destination, args.mount, args.prefix, pvm=args.pvm) if __name__ == "__main__": diff --git a/builder/scripts/test_afs_config.py b/builder/scripts/test_afs_config.py index 4d19bb37..38e6de76 100644 --- a/builder/scripts/test_afs_config.py +++ b/builder/scripts/test_afs_config.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 """AFS mount readiness and shared checkpoint profile regression tests.""" -import copy import importlib.util import os from pathlib import Path @@ -25,131 +24,41 @@ def load(name, path): CONFIG = load("afs_config", ROOT / "builder/scripts/prepare-adx-config.py") -class StandaloneCapacityTest(unittest.TestCase): - def test_default_source_is_replaced_after_full_profile_resolution(self): - original = { - "schema_version": 1, - "services": [{"id": "worker", "role": "adxlet", "config": { - "resource_source": {"kind": "auto", "disk_path": "/home/akernel", - "valid_for_seconds": 30}, - "preserved": "value", - }}], - } - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"ENABLE_TRACE": "false"}): - source = ROOT / "builder/config/adx-standalone.yaml" - target = Path(directory) / "resolved.yaml" - with patch.object(CONFIG.subprocess, "run") as run: - run.return_value.stdout = yaml.safe_dump(original) - CONFIG.configure(source, target, None, "unused", sandboxd_capacity=True) - run.assert_called_once_with( - ["/opt/adx/current/bin/adxctl", "--config", str(source), "config", "dump"], - check=True, capture_output=True, text=True, - ) - result = yaml.safe_load(target.read_text()) - expected = copy.deepcopy(original) - expected["services"][0]["config"]["resource_source"] = { - "kind": "sandboxd", "socket": "/var/run/resource.sock", - "valid_for_seconds": 30, - } - self.assertEqual(result, expected) - self.assertEqual(target.stat().st_mode & 0o777, 0o600) - - def test_custom_capacity_sources_are_preserved_without_resolving(self): - sources = ({"resource_source": {"kind": "auto", "disk_path": "/custom", - "valid_for_seconds": 7}}, - {"resource_source": {"kind": "file", "path": "/custom/capacity.json"}}, - {"resource_source": {"kind": "sandboxd", "socket": "/custom.sock", - "valid_for_seconds": 9}}, - {"capacity_file": "/custom/capacity.json"}) - with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {"ENABLE_TRACE": "false"}): - source = Path(directory) / "custom.yaml" - for explicit in (False, True): - for node in sources: - with self.subTest(explicit=explicit, node=node): - original = {"schema_version": 1} - if explicit: - original["services"] = [{"id": "worker", "role": "adxlet", "config": node}] - else: - original.update({"profile": "standalone", "service_overrides": { - "adxlet": {"config": node}}}) - source.write_text(yaml.safe_dump(original)) - with patch.object(CONFIG.subprocess, "run") as run: - CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) - run.assert_not_called() - self.assertEqual(yaml.safe_load(source.read_text()), original) - - def test_resolution_failure_preserves_in_place_source(self): - with tempfile.TemporaryDirectory() as directory: - source = Path(directory) / "source.yaml" - original = (ROOT / "builder/config/adx-standalone.yaml").read_text() - source.write_text(original) - with patch.object(CONFIG.subprocess, "run", side_effect=subprocess.CalledProcessError(1, "adxctl")): - with self.assertRaises(subprocess.CalledProcessError): - CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) - self.assertEqual(source.read_text(), original) - - def test_build_time_conversion_does_not_freeze_runtime_expressions(self): - with tempfile.TemporaryDirectory() as directory: - source = Path(directory) / "source.yaml" - original = yaml.safe_load((ROOT / "builder/config/adx-standalone.yaml").read_text()) - original["namespace"] = "${CUSTOM_NAMESPACE:-akernel}" - source.write_text(yaml.safe_dump(original)) - with patch.object(CONFIG.subprocess, "run") as run: - with self.assertRaisesRegex(ValueError, "static deployment"): - CONFIG.configure(source, source, None, "unused", sandboxd_capacity=True) - run.assert_not_called() - self.assertEqual(yaml.safe_load(source.read_text()), original) +class AfsConfigTest(unittest.TestCase): + def assert_adx_valid(self, target): + binary = os.environ.get("AKERNEL_TEST_ADXCTL") + if binary: + subprocess.run([binary, "--config", str(target), "validate"], check=True) @unittest.skipUnless(os.environ.get("AKERNEL_TEST_ADXCTL"), "requires packaged ADX CLI") - def test_packaged_cli_resolves_default_pvm_trace_and_afs_to_sandboxd(self): - binary = Path(os.environ["AKERNEL_TEST_ADXCTL"]) + def test_packaged_cli_preserves_standalone_auto_capacity_after_preparation(self): + binary = os.environ["AKERNEL_TEST_ADXCTL"] source = ROOT / "builder/config/adx-standalone.yaml" - def resolved(path): - return yaml.safe_load(subprocess.run( - [str(binary), "--config", str(path), "config", "dump"], + def resource_source(path): + deployment = yaml.safe_load(subprocess.run( + [binary, "--config", str(path), "config", "dump"], check=True, capture_output=True, text=True, ).stdout) + workers = [service for service in deployment["services"] + if service["role"] == "adxlet"] + self.assertEqual(len(workers), 1) + return workers[0]["config"]["resource_source"] + baseline = resource_source(source) + self.assertEqual(baseline["kind"], "auto") for pvm in (False, True): for trace, mount in ((False, None), (True, None), - (True, Path("/mnt/akernel-shared"))): + (False, Path("/mnt/akernel-shared"))): with self.subTest(pvm=pvm, trace=trace, mount=mount), tempfile.TemporaryDirectory() as directory: target = Path(directory) / "prepared.yaml" - baseline = Path(directory) / "baseline.yaml" - # Mock only host mount inspection; the real packaged CLI - # resolves and validates each complete deployment below. - run = subprocess.run - - def mounted(command, **kwargs): - if command[0] == "findmnt": - return subprocess.CompletedProcess(command, 0, "afs-dfs fuse rw\n", "") - return run(command, **kwargs) - - with patch.dict(os.environ, {"ENABLE_TRACE": str(trace).lower()}), patch.object(CONFIG.subprocess, "run", side_effect=mounted): - CONFIG.configure(source, baseline, mount, "test/checkpoints", pvm=pvm) - before = resolved(baseline) - CONFIG.configure(source, target, mount, "test/checkpoints", pvm=pvm, - sandboxd_capacity=True, adxctl=binary) - after = resolved(target) - self.assertNotIn("profile", yaml.safe_load(target.read_text())) - worker = next(service for service in after["services"] if service["role"] == "adxlet") - self.assertEqual(worker["config"].pop("resource_source"), { - "kind": "sandboxd", "socket": "/var/run/resource.sock", - "valid_for_seconds": 30, - }) - old_worker = next(service for service in before["services"] if service["role"] == "adxlet") - self.assertEqual(old_worker["config"].pop("resource_source")["kind"], "auto") - self.assertEqual(after, before) - subprocess.run([str(binary), "--config", str(target), "validate"], - check=True, capture_output=True, text=True) - - -class AfsConfigTest(unittest.TestCase): - def assert_adx_valid(self, target): - binary = os.environ.get("AKERNEL_TEST_ADXCTL") - if binary: - subprocess.run([binary, "--config", str(target), "validate"], check=True) + with patch.dict(os.environ, {"ENABLE_TRACE": str(trace).lower()}): + # Only mount inspection is simulated; capacity comes + # from the real packaged CLI's resolved deployments. + with patch.object(CONFIG.subprocess, "run") as run: + run.return_value.stdout = "afs-dfs fuse rw\n" + CONFIG.configure(source, target, mount, "test/checkpoints", pvm=pvm) + self.assertEqual(resource_source(target), baseline) def test_trace_preserves_explicit_services_and_only_changes_present_roles(self): for profile in ("standalone-managed-redis", "node", "coordinator", "ingress-api"): diff --git a/deploy/README.md b/deploy/README.md index df726e07..f1088328 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -19,8 +19,8 @@ application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-applicati ## Aggregate sandbox memory budget -Standalone and the direct Helm chart enable the cgroup v2 aggregate sandbox -memory policy by default for the ordinary KVM backend profile: +The direct Helm chart enables the cgroup v2 aggregate sandbox memory policy by +default for the ordinary KVM backend profile: ```toml [plugin.resource.memory_limit] @@ -36,29 +36,29 @@ without subtracting the reserve a second time. Helm and cloud Adxlet configs consume that socket. For example, effective memory of 64 GiB and this reserve produce a candidate of 48 GiB; an existing 40 GiB cap remains 40 GiB. -The bundled standalone ADX profile is resolved at image build time and its -complete capacity source is replaced with `kind: sandboxd`, using -`/var/run/resource.sock`. The PVM, trace and shared-storage variants retain this -source. Public ADX capacity follows the sandbox budget, while allocatable -decreases on resource allocation and recovers on deletion. Custom ADX -configurations with an explicitly selected source remain operator-owned; align -that source with the effective sandbox budget when enabling a parent limit. -The public API reports memory in whole MiB rounded down; parent cgroups and the -sandboxd socket retain byte precision. Compare each interface in its documented -units, including when a finite ancestor is only page-aligned. - -The default requires cgroup v2 and effective memory greater than 16 GiB, with -enough remaining capacity for the requested sandboxes. Docker/OrbStack VM limits -and visible ancestor limits participate in this calculation. Use a matching node -image built with the pinned sandboxd's `total_reserve` support. On a smaller node, -explicitly choose a smaller positive `total_reserve` or set `enabled=false` in -the sandboxd TOML before starting. Standalone reads its source config template; -Helm uses `node.config.sandboxd.config` (`core.node.config.sandboxd.config` in the -umbrella chart). Cloud profiles expose the enabled flag and reserve described -below, alongside their existing complete-config override. Values use integer -bytes or `B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not -accepted. Standard CI explicitly uses a 2 GiB reserve while keeping the policy -enabled. +Standalone keeps the policy disabled by default, with `total_reserve="16GiB"` +available for explicit configuration. Its bundled ADX profile retains the +`auto` resource source, including PVM, trace and shared-storage variants. +`auto` does not guarantee public scheduling capacity follows the sandbox parent +ceiling. Before enabling the policy, operators must configure a compatible ADX +`resource_source` and qualify memory capacity against the parent budget, +allocation and release after sandbox create/delete, and schedulable CPU. This +also applies to custom ADX configurations. The disabled default does not repair +or qualify resource-source behavior; CPU discovery is unchanged by this policy. + +Enabling the policy requires cgroup v2 and effective memory greater than the +reserve, with enough remaining capacity for the requested sandboxes. +Docker/OrbStack VM limits and visible ancestor limits participate in this +calculation. Use a matching node image built with the pinned sandboxd's +`total_reserve` support. On a smaller node, explicitly choose a smaller positive +`total_reserve` or keep `enabled=false` in the sandboxd TOML before starting. +Standalone reads its source config template; Helm uses +`node.config.sandboxd.config` (`core.node.config.sandboxd.config` in the umbrella +chart). Cloud profiles expose the enabled flag and reserve described below, +alongside their existing complete-config override. Values use integer bytes or +`B`, `KiB`, `MiB`, `GiB`, `TiB`; `16G`, percentages and `auto` are not accepted. +Verify schedulable CPU from the selected resource provider as well as memory +before rollout, including when using Helm's sandboxd source. The PVM profile keeps this policy disabled by default because enabled parent enforcement has not been qualified on its matching host. Selecting PVM overrides diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index 9e04e4fd..941559e4 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -222,12 +222,11 @@ recovery: a failing handler closes that class's advertisement/admission; successful polling reopens it. Core metadata/storage failures can still close global readiness. -Both image profiles resolve the bundled standalone deployment and select -sandboxd's resource socket for scheduling capacity. The PVM image additionally -applies `prepare-adx-config.py --pvm`, setting coordinator/adxlet -`rpc_timeout_seconds=90`. Custom or external ADX configurations must carry the -override themselves. The guest-agent startup budget remains 15 seconds; -extending an RPC cleanup budget does not authorize increasing that deadline. +The PVM image applies `prepare-adx-config.py --pvm` to its standalone profile, +setting coordinator/adxlet `rpc_timeout_seconds=90`. Custom or external ADX +configurations must carry the override themselves. The ordinary KVM profile +is unchanged and the guest-agent startup budget remains 15 seconds; extending +an RPC cleanup budget does not authorize increasing the guest startup deadline. The aggregate sandbox memory policy remains disabled by default for PVM until its finite-cap profile is qualified on the matching maintained host. Standalone @@ -235,9 +234,12 @@ can explicitly select `AKERNEL_PVM_MEMORY_LIMIT_ENABLED=true`; Helm uses `node.config.sandboxd.pvmMemoryLimitEnabled=true` (under `core` in the umbrella chart). Both default to false and apply only to the PVM backend. Select a `total_reserve` suitable for the actual L1 in its sandboxd configuration before -opting in, then run the matching bounded PVM qualification. Ordinary runsc tests -and static profile checks do not qualify this combination, and the 16 GiB -deployment reserve is not a universal host OOM safety budget. +opting in, configure a compatible ADX `resource_source`, and qualify memory +capacity, allocation/release accounting and schedulable CPU on that host. The +bundled standalone profile retains `auto`, which does not guarantee capacity +follows the sandbox parent limit. Ordinary runsc tests and static profile +checks do not qualify this combination, and the 16 GiB deployment reserve is +not a universal host OOM safety budget. ## Isolated qualification workflow diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index ed2981ac..88b8b76a 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -58,17 +58,16 @@ for inspection and retry. Data and credentials survive stopping. For an independent data profile, set an absolute `AKERNEL_STANDALONE_DATA_DIR` for every command. Only one standalone node can run per Docker daemon. -The ordinary KVM backend profile enables the aggregate sandbox memory limit -with a 16 GiB reserve. -It requires cgroup v2 and effective memory above that reserve, including any -Docker/OrbStack VM or visible ancestor limit. Before starting on a smaller -machine, edit `[plugin.resource.memory_limit]` in -`deploy/standalone/config/sandboxd_config.toml` to select a smaller positive -`total_reserve` or explicitly disable the policy. Editing only the generated -data-directory TOML is insufficient because startup regenerates it. This also -applies to `make standalone-source-test`. See +Standalone keeps the aggregate sandbox memory limit disabled by default, with +`total_reserve="16GiB"` available for explicit configuration. To enable it, edit +`[plugin.resource.memory_limit]` in +`deploy/standalone/config/sandboxd_config.toml` before startup. Qualify cgroup v2 +and effective memory above the selected positive reserve, including any +Docker/OrbStack VM or visible ancestor limit, with room for sandbox requests. +Editing only the generated data-directory TOML is insufficient because startup +regenerates it. This also applies to `make standalone-source-test`. See [the memory budget and upgrade contract](../README.md#aggregate-sandbox-memory-budget) -before applying the new default to an existing profile. +before enabling the policy on an existing profile. Selecting `AKERNEL_FIRECRACKER_BACKEND=pvm` keeps the parent limit disabled by default, including when the source TOML has `enabled=true`. PVM parent-limit @@ -78,13 +77,14 @@ positive `total_reserve` in the source TOML before startup. The switch accepts only `true` or `false`, defaults to `false`, and affects only PVM. The maintained PVM CI fixture keeps it disabled. -The bundled ADX configuration takes scheduling capacity from sandboxd's -`/var/run/resource.sock`, including PVM and trace/shared-storage variants. -The image build resolves the default profile before replacing the entire -capacity source. Custom `AKERNEL_ADX_CONFIG` files retain their selected source; -operators enabling the parent limit must align custom capacity reporting with -that limit. Standalone CI checks both the parent/socket budget and public ADX -capacity, plus resource allocation and release after sandbox create/delete. +The bundled ADX profile retains its `auto` resource source, including PVM and +trace/shared-storage variants. `auto` does not guarantee that public scheduling +capacity follows the sandbox parent ceiling. Before enabling the policy, +operators must configure and qualify a compatible ADX `resource_source`, whether +using the bundled profile or a custom `AKERNEL_ADX_CONFIG`. Verify memory +capacity against the parent budget, allocation and release after sandbox +create/delete, and schedulable CPU from that source. The disabled default does +not repair or qualify resource-source behavior. ### Runtime selection diff --git a/deploy/standalone/config/sandboxd_config.toml b/deploy/standalone/config/sandboxd_config.toml index 354f88f8..c6c0fea2 100644 --- a/deploy/standalone/config/sandboxd_config.toml +++ b/deploy/standalone/config/sandboxd_config.toml @@ -29,8 +29,8 @@ max_instance_num=1000 enable_cache_in_stock_mode=true [plugin.resource.memory_limit] -enabled = true -# Requires effective memory above this reserve; tune for the deployment. +enabled = false +# Before enabling, qualify scheduling capacity and tune this deployment reserve. total_reserve = "16GiB" [plugin.runtime] diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index 6d2f6c08..93cf92c6 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -107,10 +107,10 @@ def assert_memory_config(self, enabled, reserve): ) self.assertEqual(config["plugin.image"]["cgroup_memory_limit"], '"64GiB"') - def test_generated_config_enables_parent_limit_with_16gib_reserve(self): + def test_generated_config_disables_parent_limit_with_16gib_reserve(self): result = self.generate(AKERNEL_ENABLE_RUNC="false") self.assertEqual(result.returncode, 0, result.stderr) - self.assert_memory_config("true", "16GiB") + self.assert_memory_config("false", "16GiB") def test_generated_config_preserves_explicit_memory_policy_overrides(self): original = self.template.read_text() @@ -144,7 +144,7 @@ def test_pvm_parent_limit_requires_explicit_opt_in_and_preserves_reserve(self): original = self.template.read_text().replace('total_reserve = "16GiB"', 'total_reserve = "2GiB"') for key in ("enabled", '"enabled"', "'enabled'"): for source_enabled in ("false", "true"): - self.template.write_text(original.replace("enabled = true", f"{key} = {source_enabled}", 1)) + self.template.write_text(original.replace("enabled = false", f"{key} = {source_enabled}", 1)) for enabled in ("false", "true"): with self.subTest(key=key, source_enabled=source_enabled, enabled=enabled): result = self.generate( @@ -155,10 +155,13 @@ def test_pvm_parent_limit_requires_explicit_opt_in_and_preserves_reserve(self): self.assert_memory_config(enabled, "2GiB") def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): - self.template.write_text(self.template.read_text().replace("enabled = true", "enabled = false", 1)) - result = self.generate(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true") - self.assertEqual(result.returncode, 0, result.stderr) - self.assert_memory_config("false", "16GiB") + original = self.template.read_text() + for enabled in ("false", "true"): + with self.subTest(enabled=enabled): + self.template.write_text(original.replace("enabled = false", f"enabled = {enabled}", 1)) + result = self.generate(AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(enabled, "16GiB") def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): original = self.template.read_text() @@ -166,7 +169,7 @@ def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", "", original, ) - without_enabled = original.replace("enabled = true\n", "", 1) + without_enabled = original.replace("enabled = false\n", "", 1) for custom, has_table in ((without_table, False), (without_enabled, True)): with self.subTest(has_table=has_table): self.template.write_text(custom) @@ -195,7 +198,7 @@ def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): "enabled = true\n'enabled' = false"): for opt_in in ("false", "true"): with self.subTest(field=field, opt_in=opt_in): - self.template.write_text(original.replace("enabled = true", field, 1)) + self.template.write_text(original.replace("enabled = false", field, 1)) result = self.generate( AKERNEL_FIRECRACKER_BACKEND="pvm", AKERNEL_PVM_MEMORY_LIMIT_ENABLED=opt_in, diff --git a/sdk/python/tests/e2e/README.md b/sdk/python/tests/e2e/README.md index d8906261..366743aa 100644 --- a/sdk/python/tests/e2e/README.md +++ b/sdk/python/tests/e2e/README.md @@ -17,14 +17,6 @@ A public SDK Full run is not the entire platform deployment acceptance suite. Most cases request 1 CPU / 2 GiB per Sandbox; concurrent fixtures need capacity for multiple instances. -The `sdk-contracts` group runs `SandboxPublicContractIntegrationTest` for every -runtime. Parent memory-budget qualification is a separate -`MemoryBudgetIntegrationTest` in the same module, discovered by standard CI's -standalone unittest command with `AKERNEL_TEST_EXPECT_MEMORY_BYTES` set to the -measured finite parent limit. It also checks CPU when -`AKERNEL_TEST_EXPECT_CPU_MILLIS` is set. The PVM fixture leaves parent enforcement -disabled and does not include this qualification in its SDK smoke coverage. - ## Run Install the SDK from this checkout and configure its diff --git a/sdk/python/tests/e2e/run.py b/sdk/python/tests/e2e/run.py index e4960467..c4d6cbf6 100644 --- a/sdk/python/tests/e2e/run.py +++ b/sdk/python/tests/e2e/run.py @@ -31,7 +31,7 @@ class Case: Case( "sdk-contracts", "l0", - ("-m", "unittest", "-v", "tests.e2e.standalone.test_sdk_contracts.SandboxPublicContractIntegrationTest"), + ("-m", "unittest", "-v", "tests.e2e.standalone.test_sdk_contracts"), 300, ), Case( diff --git a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py index cbc7552d..8216a91c 100644 --- a/sdk/python/tests/e2e/standalone/test_sdk_contracts.py +++ b/sdk/python/tests/e2e/standalone/test_sdk_contracts.py @@ -16,7 +16,6 @@ import os import tempfile -import time import unittest import uuid from pathlib import Path @@ -30,76 +29,6 @@ ) _RUNTIME = os.environ.get("AKERNEL_TEST_RUNTIME", "runsc") _IMAGE = os.environ.get("AKERNEL_TEST_IMAGE") or None -_MEMORY_BUDGET = os.environ.get("AKERNEL_TEST_EXPECT_MEMORY_BYTES") - - -@unittest.skipUnless( - _ENABLED and _MEMORY_BUDGET, - "set the SDK integration environment and AKERNEL_TEST_EXPECT_MEMORY_BYTES", -) -class MemoryBudgetIntegrationTest(unittest.TestCase): - """Check the public scheduler against a measured single-node parent cap.""" - - def test_capacity_and_allocation_follow_the_parent_memory_budget(self): - # The public resource API rounds byte capacities down to whole MiB. - expected = int(_MEMORY_BUDGET) // (1024 * 1024) - self.assertGreater(expected, 0) - cpu_budget = os.environ.get("AKERNEL_TEST_EXPECT_CPU_MILLIS") - expected_cpu = int(cpu_budget) if cpu_budget else None - request_cpu = min(1000, expected_cpu) if expected_cpu is not None else 1000 - self.assertGreater(request_cpu, 0) - - def wait_for_memory(allocatable=None, allocatable_cpu=None): - deadline = time.monotonic() + 60 - last = [] - while time.monotonic() < deadline: - last = resources() - self.assertLessEqual(len(last), 1, "requires an isolated single node") - if last: - node = last[0] - capacity = node.capacity.get("Memory", -1) - available = node.allocatable.get("Memory", -1) - cpu_matches = expected_cpu is None or ( - node.capacity.get("CPU", -1) == expected_cpu - and ( - allocatable_cpu is None - or node.allocatable.get("CPU", -1) == allocatable_cpu - ) - ) - if abs(capacity - expected) < 0.000001 and ( - allocatable is None - or abs(available - allocatable) < 0.000001 - ) and cpu_matches: - return node - time.sleep(0.5) - self.fail( - f"expected memory capacity={expected}, allocatable={allocatable}; " - f"CPU capacity={expected_cpu}, allocatable={allocatable_cpu}; " - f"last resource view: {last}" - ) - - before = wait_for_memory() - available = before.allocatable["Memory"] - self.assertGreaterEqual(available, 1024) - available_cpu = ( - before.allocatable["CPU"] if expected_cpu is not None else None - ) - if available_cpu is not None: - self.assertGreaterEqual(available_cpu, request_cpu) - sandbox = Sandbox(cpu=request_cpu, memory=1024, runtime=_RUNTIME, image=_IMAGE) - try: - self.assertEqual( - sandbox.commands.run("printf memory-budget").stdout, "memory-budget" - ) - running = wait_for_memory( - available - 1024, - available_cpu - request_cpu if available_cpu is not None else None, - ) - self.assertEqual(running.id, before.id) - finally: - sandbox.kill() - released = wait_for_memory(available, available_cpu) - self.assertEqual(released.id, before.id) @unittest.skipUnless( diff --git a/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py b/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py deleted file mode 100644 index 1a5c7873..00000000 --- a/sdk/python/tests/unit/test_e2e_memory_budget_fixture.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Public resource units used by the standalone memory-budget regression.""" - -import os -import unittest -from unittest.mock import patch - -from akernel_sdk import NodeInfo -from tests.e2e.standalone import test_sdk_contracts as fixture - - -class MemoryBudgetFixtureTest(unittest.TestCase): - def test_page_aligned_parent_caps_follow_whole_mib_allocation_and_release(self): - public_capacity = 6144.0 - cases = [(extra, None) for extra in (0, 4096, 1024 * 1024 - 4096)] - cases.extend([(4096, 500), (4096, 1500)]) - for extra_bytes, cpu_budget in cases: - with self.subTest(extra_bytes=extra_bytes, cpu_budget=cpu_budget): - maximum = 6 * 1024**3 + extra_bytes - request_cpu = min(1000, cpu_budget) if cpu_budget is not None else 1000 - - def node(available, cpu=None, budget=cpu_budget): - return NodeInfo( - id="single-node", - status=0, - capacity={ - "Memory": public_capacity, - **({"CPU": budget} if budget is not None else {}), - }, - allocatable={ - "Memory": available, - **({"CPU": cpu} if cpu is not None else {}), - }, - labels={}, - ) - - case = fixture.MemoryBudgetIntegrationTest( - "test_capacity_and_allocation_follow_the_parent_memory_budget" - ) - with ( - patch.object(fixture, "_MEMORY_BUDGET", str(maximum)), - patch.dict( - os.environ, - { - "AKERNEL_TEST_EXPECT_CPU_MILLIS": - str(cpu_budget) if cpu_budget else "" - }, - ), - patch.object( - fixture, - "resources", - side_effect=[ - [node(public_capacity, cpu_budget)], - [node( - public_capacity - 1024, - cpu_budget - request_cpu if cpu_budget else None, - )], - [node(public_capacity, cpu_budget)], - ], - ) as resources, - patch.object(fixture, "Sandbox") as sandbox, - patch.object( - fixture.time, - "sleep", - side_effect=AssertionError( - "public memory should match immediately" - ), - ), - ): - command = sandbox.return_value.commands.run.return_value - command.stdout = "memory-budget" - case.test_capacity_and_allocation_follow_the_parent_memory_budget() - - self.assertEqual(resources.call_count, 3) - sandbox.assert_called_once_with( - cpu=request_cpu, - memory=1024, - runtime=fixture._RUNTIME, - image=fixture._IMAGE, - ) - sandbox.return_value.kill.assert_called_once_with() - - -if __name__ == "__main__": - unittest.main() diff --git a/src/sandboxd b/src/sandboxd index a4ba0cf6..fbe69e56 160000 --- a/src/sandboxd +++ b/src/sandboxd @@ -1 +1 @@ -Subproject commit a4ba0cf6aedef1431371b8a66034963cf8685b4b +Subproject commit fbe69e566859718414a207fa780b8dab363b9381 From 47d8563687ee738e39875b3aeb922049eae330f2 Mon Sep 17 00:00:00 2001 From: "wanghongtao.wangho" Date: Mon, 9 Nov 2026 10:37:22 +0800 Subject: [PATCH 9/9] fix(deploy): enforce PVM memory switches on parsed TOML Resolve Helm policies by their parsed keys so equivalent table forms cannot bypass an explicit opt-out. Verify standalone rewrites against the full parsed input and reject unsupported overrides before writing. Exercise both switch values through the deployment entry points and retain compatibility with omitted disabled policies and KVM. Signed-off-by: wanghongtao.wangho --- .github/workflows/ci.yml | 1 + AGENTS.md | 4 + .../charts/core/scripts/pvm-memory-limit.tpl | 38 +++++++++ .../charts/core/templates/node/configmap.yaml | 19 ++--- .../tests/test_shared_checkpoint_storage.py | 44 ++++++++++- deploy/pvm-runtime.md | 9 +++ deploy/standalone/README.md | 5 ++ deploy/standalone/prepare-pvm-memory.py | 78 +++++++++++++++++++ deploy/standalone/start.sh | 26 +++---- deploy/standalone/test_start_config.py | 55 +++++++++++-- 10 files changed, 245 insertions(+), 34 deletions(-) create mode 100644 deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl create mode 100644 deploy/standalone/prepare-pvm-memory.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b793ca8..66215efb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,7 @@ jobs: - name: Test standalone deployment helpers run: | + python3 -m pip install 'tomli>=2,<3' make deploy-standalone-test make build-helper-test adx-integration-contracts: diff --git a/AGENTS.md b/AGENTS.md index a69d065b..decf6322 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -264,6 +264,10 @@ PVM also keeps the policy disabled until explicitly selected with `node.config.sandboxd.pvmMemoryLimitEnabled=true` in Helm. Size the reserve for the selected L1 and qualify the finite-cap and scheduling combination on the matching PVM host before rollout; ordinary runsc tests do not qualify PVM. +Validate PVM memory switches against parsed TOML. PVM chart rendering requires +Helm 3.17 or newer, and standalone uses Python 3.11 or Python 3.10 with +`tomli`. Reject a standalone rewrite unless the parsed configuration matches +the requested policy and preserves the other fields. Adding or lowering a parent cap requires an empty sandbox subtree; an existing stricter cap is retained. This aggregate sandbox cap does not bound all node components or guarantee avoidance of host OOM. diff --git a/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl b/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl new file mode 100644 index 00000000..05c33bb2 --- /dev/null +++ b/deploy/akernel/charts/core/scripts/pvm-memory-limit.tpl @@ -0,0 +1,38 @@ +{{- $config := fromToml .config -}} +{{- if hasKey $config "Error" -}} +{{- fail "PVM sandboxd.config must be valid TOML with no duplicate fields" -}} +{{- end -}} +{{- $table := $config -}} +{{- range $key := list "plugin" "resource" "memory_limit" -}} +{{- /* sandboxd also accepts uppercase aliases for its struct fields. */ -}} +{{- if hasKey $table (upper $key) -}} +{{- fail "PVM memory policy requires lowercase plugin.resource.memory_limit keys" -}} +{{- end -}} +{{- if hasKey $table $key -}} +{{- $table = get $table $key -}} +{{- if not (kindIs "map" $table) -}} +{{- fail (printf "PVM memory policy requires %s to be a TOML table" $key) -}} +{{- end -}} +{{- else -}} +{{- $table = dict -}} +{{- end -}} +{{- end -}} +{{- if hasKey $table "enabled" -}} +{{- if not (kindIs "bool" (get $table "enabled")) -}} +{{- fail "PVM memory policy requires a boolean enabled field" -}} +{{- end -}} +{{- if ne (get $table "enabled") .enabled -}} +{{- $_ := set $table "enabled" .enabled -}} +{{- $rendered := toToml $config -}} +{{- if or (not $rendered) (hasKey (fromToml $rendered) "Error") -}} +{{- fail "Could not render the PVM sandboxd configuration as TOML" -}} +{{- end -}} +{{- $rendered -}} +{{- else -}} +{{- .config -}} +{{- end -}} +{{- else if .enabled -}} +{{- fail "PVM memory policy requires a boolean enabled field when opting in" -}} +{{- else -}} +{{- .config -}} +{{- end -}} diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index 010f97c0..33ab789d 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -65,18 +65,6 @@ data: {{- if not (and (contains "[plugin.runtime.firecracker]" $sandboxdConfig) (contains "firecracker=\"/usr/local/bin/firecracker\"" $sandboxdConfig)) }} {{- fail "PVM selection requires the standard Firecracker config section and binary entry" }} {{- end }} -{{- $memorySection := regexFind "(?ms)^[\\t ]*\\[plugin\\.resource\\.memory_limit\\][^\\n]*\\n.*?(?:\\n[\\t ]*\\[|\\z)" $sandboxdConfig }} -{{- $memoryEnabledPattern := "(?m)^[\\t ]*(enabled|\"enabled\"|'enabled')[\\t ]*=[\\t ]*(true|false)[\\t ]*(#[^\\n]*)?$" }} -{{- $memoryEnabledCount := len (regexFindAll $memoryEnabledPattern $memorySection -1) }} -{{- $memoryEnabledFields := len (regexFindAll "(?m)^[\\t ]*(enabled|\"enabled\"|'enabled')[\\t ]*=" $memorySection -1) }} -{{- if or (ne $memoryEnabledFields $memoryEnabledCount) (gt $memoryEnabledCount 1) (and (eq $pvmMemoryLimitEnabled "true") (ne $memoryEnabledCount 1)) }} -{{- fail "PVM selection requires at most one boolean enabled field under [plugin.resource.memory_limit], and exactly one when opting in" }} -{{- end }} -{{- /* Older custom profiles may omit the optional, disabled memory policy. */}} -{{- if eq $memoryEnabledCount 1 }} -{{- $pvmMemorySection := regexReplaceAll $memoryEnabledPattern $memorySection (printf "enabled = %s" $pvmMemoryLimitEnabled) }} -{{- $sandboxdConfig = replace $memorySection $pvmMemorySection $sandboxdConfig }} -{{- end }} {{- $sandboxdConfig = regexReplaceAll "(?m)^([\\t ]*)firecracker=" $sandboxdConfig "${1}firecracker-pvm=" }} {{- $sandboxdConfig = replace "[plugin.runtime.firecracker]" "[plugin.runtime.firecracker_pvm]" $sandboxdConfig }} {{- $sandboxdConfig = regexReplaceAll "(?m)^[\\t ]*kata=.*$" $sandboxdConfig "# Kata is disabled on the PVM node profile." }} @@ -99,6 +87,13 @@ data: {{- end }} {{- $sandboxdConfig = replace $chunkDbMarker (printf "chunk_db_size=%s" ($chunkDbSize | quote)) $sandboxdConfig }} {{- end }} +{{- if eq $firecrackerBackend "pvm" }} +{{- if not (semverCompare ">=3.17.0" .Capabilities.HelmVersion.Version) }} +{{- fail "PVM configuration requires Helm 3.17 or newer for TOML validation" }} +{{- end }} +{{- /* Parse after consuming template markers. Bind fromToml only on the PVM path. */}} +{{- $sandboxdConfig = tpl (.Files.Get "scripts/pvm-memory-limit.tpl") (dict "Template" .Template "config" $sandboxdConfig "enabled" (eq $pvmMemoryLimitEnabled "true")) }} +{{- end }} {{ $sandboxdConfig | indent 4 }} registry.json: | diff --git a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py index 1ed6c6bd..29467266 100644 --- a/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py +++ b/deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py @@ -108,6 +108,46 @@ def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) self.assert_memory_config(config, False, "16GiB") + def test_pvm_parent_switch_accepts_equivalent_toml_policy_forms(self): + default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] + header = "[plugin.resource.memory_limit]" + section = r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "pvm-sandboxd.toml" + for enabled, source in (("false", "true"), ("true", "false")): + canonical = default.replace("enabled = true", f"enabled = {source}", 1) + forms = { + "canonical": canonical, + "quoted": canonical.replace(header, '[plugin."resource".memory_limit]'), + "escaped": canonical.replace(header, r'[plugin."res\u006furce".memory_limit]'), + "inline": re.sub( + section, + f'memory_limit = {{ enabled = {source}, total_reserve = "16GiB" }}\n\n', + canonical, + ), + "dotted": re.sub( + section, + f'memory_limit.enabled = {source}\nmemory_limit.total_reserve = "16GiB"\n\n', + canonical, + ), + } + expected = None + for form, candidate in forms.items(): + with self.subTest(enabled=enabled, form=form): + self.assertEqual(tomllib.loads(candidate), tomllib.loads(canonical)) + path.write_text(candidate) + resources = render( + "node.config.sandboxd.firecrackerBackend=pvm", + f"node.config.sandboxd.pvmMemoryLimitEnabled={enabled}", + set_files={"node.config.sandboxd.config": path}, + ) + config = tomllib.loads(resources[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"]) + self.assert_memory_config(config, enabled == "true", "16GiB") + if expected is None: + expected = config + else: + self.assertEqual(config, expected) + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] without_table = re.sub( @@ -138,7 +178,7 @@ def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): "node.config.sandboxd.pvmMemoryLimitEnabled=true", set_files={"node.config.sandboxd.config": path}, ) - self.assertIn("exactly one when opting in", error.exception.stderr) + self.assertIn("boolean enabled field when opting in", error.exception.stderr) def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): default = render()[("ConfigMap", "akernel-node-config")]["data"]["sandboxd_config.toml"] @@ -156,7 +196,7 @@ def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): f"node.config.sandboxd.pvmMemoryLimitEnabled={opt_in}", set_files={"node.config.sandboxd.config": path}, ) - self.assertIn("boolean enabled field", error.exception.stderr) + self.assertRegex(error.exception.stderr, "boolean enabled field|duplicate fields") def test_pvm_rejects_invalid_memory_opt_in(self): for value in ("1", "TRUE", "yes", ""): diff --git a/deploy/pvm-runtime.md b/deploy/pvm-runtime.md index 941559e4..6ce52f59 100644 --- a/deploy/pvm-runtime.md +++ b/deploy/pvm-runtime.md @@ -241,6 +241,15 @@ follows the sandbox parent limit. Ordinary runsc tests and static profile checks do not qualify this combination, and the 16 GiB deployment reserve is not a universal host OOM safety budget. +PVM configuration requires Helm 3.17 or newer for chart rendering, or Python +3.11 or newer for the standalone launcher. Standalone also accepts Python 3.10 +with `tomli` installed in the launcher's Python environment. The generators +validate the TOML before writing the configuration. Helm can override quoted +and inline tables. For standalone, use the standard +`[plugin.resource.memory_limit]` section with a boolean `enabled` field when +the switch needs to change that value. Standalone rejects an override it +cannot apply and leaves the previous generated configuration in place. + ## Isolated qualification workflow Ordinary sandboxd hardware-KVM CI and AKernel's runsc-only standalone image do diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 88b8b76a..84ec6995 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -77,6 +77,11 @@ positive `total_reserve` in the source TOML before startup. The switch accepts only `true` or `false`, defaults to `false`, and affects only PVM. The maintained PVM CI fixture keeps it disabled. +For PVM, use Python 3.11 or newer, or install `tomli` in the Python 3.10 +environment used by the launcher. To override a custom memory policy, put its +boolean `enabled` field in the standard `[plugin.resource.memory_limit]` +section. See the [PVM deployment guide](../pvm-runtime.md#deployment-and-storage-profile). + The bundled ADX profile retains its `auto` resource source, including PVM and trace/shared-storage variants. `auto` does not guarantee that public scheduling capacity follows the sandbox parent ceiling. Before enabling the policy, diff --git a/deploy/standalone/prepare-pvm-memory.py b/deploy/standalone/prepare-pvm-memory.py new file mode 100644 index 00000000..b2dfd9de --- /dev/null +++ b/deploy/standalone/prepare-pvm-memory.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026 Ant Group Corporation. +# SPDX-License-Identifier: Apache-2.0 +"""Apply the standalone PVM switch to a validated sandboxd configuration.""" + +import argparse +from pathlib import Path +import re +import sys + +try: + import tomllib +except ModuleNotFoundError: + try: + import tomli as tomllib + except ModuleNotFoundError: + raise SystemExit("PVM configuration requires Python 3.11+ or tomli on Python 3.10") from None + + +MEMORY_TABLE = re.compile( + r"(?ms)^[\t ]*\[plugin\.resource\.memory_limit\][\t ]*(?:#[^\n]*)?\n" + r"(?P.*?)(?=^[\t ]*\[|\Z)" +) +ENABLED = re.compile( + r"""(?m)^([\t ]*)(?:enabled|"enabled"|'enabled')[\t ]*=[\t ]*""" + r"(?:true|false)(?=[\t ]*(?:#|$))" +) + + +def configure(source: str, enabled: bool) -> str: + config = tomllib.loads(source) + table = config + for key in ("plugin", "resource", "memory_limit"): + # Match the sandboxd decoder's acceptance of uppercase struct aliases. + if key.upper() in table: + raise ValueError("PVM memory policy requires lowercase plugin.resource.memory_limit keys") + table = table.get(key, {}) + if not isinstance(table, dict): + raise ValueError(f"PVM memory policy requires {key} to be a TOML table") + + if "enabled" not in table: + if enabled: + raise ValueError("PVM memory policy requires a boolean enabled field when opting in") + return source + current = table["enabled"] + if not isinstance(current, bool): + raise ValueError("PVM memory policy requires a boolean enabled field") + if current == enabled: + return source + + # Verify the full parsed result so a lookalike line in a string cannot + # satisfy the override or change unrelated fields. + table["enabled"] = enabled + for match in MEMORY_TABLE.finditer(source): + body = ENABLED.sub(r"\1enabled = " + str(enabled).lower(), match["body"]) + rendered = source[:match.start("body")] + body + source[match.end("body"):] + if tomllib.loads(rendered) == config: + return rendered + raise ValueError( + "Use [plugin.resource.memory_limit] with a boolean enabled field " + "to override the PVM memory policy" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path) + parser.add_argument("enabled", choices=("true", "false")) + args = parser.parse_args() + try: + sys.stdout.write(configure(args.config.read_text(), args.enabled == "true")) + except tomllib.TOMLDecodeError: + parser.exit(1, "PVM sandboxd configuration must be valid TOML with no duplicate fields\n") + except ValueError as error: + parser.exit(1, f"{error}\n") + + +if __name__ == "__main__": + main() diff --git a/deploy/standalone/start.sh b/deploy/standalone/start.sh index f3ab9a98..bc30c74b 100755 --- a/deploy/standalone/start.sh +++ b/deploy/standalone/start.sh @@ -199,6 +199,10 @@ check_prerequisites() { exit 1 ;; esac + if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]] && + ! python3 "${SCRIPT_DIR}/prepare-pvm-memory.py" --help >/dev/null; then + exit 1 + fi if [[ "${DATA_DIR}" != /* ]]; then log_error "AKERNEL_STANDALONE_DATA_DIR must be an absolute path" exit 1 @@ -325,6 +329,7 @@ configure_gpu() { configure_network() { local config_tmp="${SANDBOXD_CONFIG_FILE}.tmp" + local pvm_config local sed_args=( -E -e "s/^[[:space:]]*nat_backend[[:space:]]*=.*/nat_backend=\"${AKERNEL_NAT_BACKEND}\"/" @@ -345,23 +350,12 @@ configure_network() { log_error "PVM selection requires the standard Firecracker config section and binary entry" exit 1 fi - # Older custom profiles may omit the optional, disabled memory policy. - if ! awk -v opt_in="${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}" ' - /^[[:space:]]*\[/ { - in_memory = ($0 ~ /^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/) - } - in_memory && /^[[:space:]]*(enabled|"enabled"|\047enabled\047)[[:space:]]*=/ { fields++ } - in_memory && /^[[:space:]]*(enabled|"enabled"|\047enabled\047)[[:space:]]*=[[:space:]]*(true|false)[[:space:]]*(#.*)?$/ { booleans++ } - END { exit !(fields == booleans && booleans <= 1 && (opt_in == "false" || booleans == 1)) } - ' "${CONFIG_DIR}/sandboxd_config.toml"; then - log_error "PVM selection requires at most one boolean enabled field under [plugin.resource.memory_limit], and exactly one when opting in" - exit 1 - fi + pvm_config="$(python3 "${SCRIPT_DIR}/prepare-pvm-memory.py" \ + "${CONFIG_DIR}/sandboxd_config.toml" "${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}")" || exit 1 sed_args+=( -e 's/^firecracker=/firecracker-pvm=/' -e 's/^\[plugin.runtime.firecracker\]$/[plugin.runtime.firecracker_pvm]/' -e '/^kata=/d' - -e "/^[[:space:]]*\[plugin\.resource\.memory_limit\][[:space:]]*(#.*)?$/,/^[[:space:]]*\[/s/^([[:space:]]*)(enabled|\"enabled\"|'enabled')[[:space:]]*=.*/\1enabled = ${AKERNEL_PVM_MEMORY_LIMIT_ENABLED-false}/" ) fi @@ -424,7 +418,11 @@ configure_network() { -e "s|^[[:space:]]*# AKERNEL_CHUNK_DB_SIZE[[:space:]]*$|chunk_db_size=\"${AKERNEL_CHUNK_DB_SIZE}\"|" ) fi - sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" + if [[ "${AKERNEL_FIRECRACKER_BACKEND}" == pvm ]]; then + sed "${sed_args[@]}" <<< "${pvm_config}" > "${config_tmp}" + else + sed "${sed_args[@]}" "${CONFIG_DIR}/sandboxd_config.toml" > "${config_tmp}" + fi mv "${config_tmp}" "${SANDBOXD_CONFIG_FILE}" if [[ "${AKERNEL_ENABLE_RUNC}" == "true" ]]; then diff --git a/deploy/standalone/test_start_config.py b/deploy/standalone/test_start_config.py index 93cf92c6..4e8f0646 100644 --- a/deploy/standalone/test_start_config.py +++ b/deploy/standalone/test_start_config.py @@ -11,6 +11,11 @@ import tempfile import unittest +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib + STANDALONE_DIR = Path(__file__).resolve().parent DIRECT_MARKER = "# AKERNEL_DIRECT_RESOLVER" @@ -96,10 +101,9 @@ def assert_rejected_without_replacing_config(self, message): def assert_memory_config(self, enabled, reserve): config = self.read_sections() - self.assertEqual( - dict(config["plugin.resource.memory_limit"]), - {"enabled": enabled, "total_reserve": f'"{reserve}"'}, - ) + policy = tomllib.loads(self.config_output.read_text())["plugin"]["resource"]["memory_limit"] + self.assertEqual(policy, {"enabled": enabled == "true", "total_reserve": reserve}) + self.assertIs(policy["enabled"], enabled == "true") self.assertEqual(config["plugin.resource"]["cgroup_root_name"], '"/akernel"') self.assertEqual(config["plugin.resource"]["max_instance_num"], "1000") self.assertEqual( @@ -163,6 +167,45 @@ def test_pvm_memory_opt_in_does_not_override_kvm_memory_policy(self): self.assertEqual(result.returncode, 0, result.stderr) self.assert_memory_config(enabled, "16GiB") + def test_pvm_rejects_policy_rewrites_without_a_standard_memory_table(self): + original = self.template.read_text() + header = "[plugin.resource.memory_limit]" + for form in ("quoted", "inline", "escaped", "dotted"): + for enabled, source in (("false", "true"), ("true", "false")): + with self.subTest(form=form, enabled=enabled): + canonical = original.replace("enabled = false", f"enabled = {source}", 1) + if form == "quoted": + custom = canonical.replace(header, '[plugin."resource".memory_limit]') + elif form == "escaped": + custom = canonical.replace(header, r'[plugin."res\u006furce".memory_limit]') + else: + if form == "inline": + policy = f'memory_limit = {{ enabled = {source}, total_reserve = "16GiB" }}' + else: + policy = f'memory_limit.enabled = {source}\nmemory_limit.total_reserve = "16GiB"' + custom = re.sub( + r"(?ms)^\[plugin\.resource\.memory_limit\]\n.*?(?=^\[|\Z)", + policy + "\n\n", canonical, + ) + self.assertEqual(tomllib.loads(custom), tomllib.loads(canonical)) + self.template.write_text(custom) + self.config_output.write_text("existing config\n") + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=enabled, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Use [plugin.resource.memory_limit]", result.stderr) + self.assertEqual(self.config_output.read_text(), "existing config\n") + self.assertFalse(self.resolver_output.exists()) + result = self.generate( + AKERNEL_FIRECRACKER_BACKEND="pvm", + AKERNEL_PVM_MEMORY_LIMIT_ENABLED=source, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_memory_config(source, "16GiB") + self.resolver_output.unlink() + def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): original = self.template.read_text() without_table = re.sub( @@ -188,7 +231,7 @@ def test_pvm_preserves_omitted_disabled_memory_policy_but_rejects_opt_in(self): AKERNEL_PVM_MEMORY_LIMIT_ENABLED="true", ) self.assertNotEqual(result.returncode, 0) - self.assertIn("exactly one when opting in", result.stderr) + self.assertIn("boolean enabled field when opting in", result.stderr) self.assertEqual(self.config_output.read_text(), generated) def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): @@ -204,7 +247,7 @@ def test_pvm_rejects_invalid_or_duplicate_memory_enabled_fields(self): AKERNEL_PVM_MEMORY_LIMIT_ENABLED=opt_in, ) self.assertNotEqual(result.returncode, 0) - self.assertIn("boolean enabled field", result.stderr) + self.assertRegex(result.stderr, "boolean enabled field|duplicate fields") self.assertFalse(self.config_output.exists()) def test_pvm_rejects_invalid_memory_opt_in(self):