From ca0cacc55beb1ff273441c008c761f5e48ec54b9 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Mon, 10 Aug 2026 21:54:58 -0700 Subject: [PATCH 1/2] microvm: move cloud-hypervisor to v53.0 cloud-hypervisor publishes minor releases and rarely backports to a patch branch, so staying on v52.0 means running a hypervisor that receives nothing. v53.0 needs the restore mode chosen for it, which the preceding commits do: its userfaultfd restore handler background-prefaults every registered page, which starves the guest badly enough that an actor never passes its readiness probe. Restoring eagerly avoids that, and the runtime now selects it from the version the VMM reports. The cost is memory: a restored idle actor holds its whole snapshot resident rather than the pages it touches, ~158MiB against ~16MiB on the counter demo, because prefaulting fills whatever is registered. Nothing configurable recovers that; the ways out are a smaller guest working set or in-VMM snapshot chaining upstream (cloud-hypervisor#8645). Suspend cost and per-actor disk are unchanged, at ~0.32s and ~158MiB. --- hack/microvm-assets/assemble.sh | 4 ++-- manifests/microvm/sandboxconfig-microvm.yaml.tmpl | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/microvm-assets/assemble.sh b/hack/microvm-assets/assemble.sh index ab0c3e7a14..cb01d9ae63 100755 --- a/hack/microvm-assets/assemble.sh +++ b/hack/microvm-assets/assemble.sh @@ -34,7 +34,7 @@ # publishes a prebuilt static binary for x86_64 only; arm64 builds from the release # tag, which needs rust (rustup) + libcap-ng-dev libseccomp-dev pkg-config. # -# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.0.0), CH_VER (v52.0), +# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.0.0), CH_VER (v53.0), # OUT (default ./bin/microvm-assets/$ARCH, under the gitignored bin/). set -o errexit -o nounset -o pipefail @@ -43,7 +43,7 @@ ROOT="$(git rev-parse --show-toplevel)" ARCH="${ARCH:-arm64}" KATA_VER="${KATA_VER:-4.0.0}" -CH_VER="${CH_VER:-v52.0}" +CH_VER="${CH_VER:-v53.0}" OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 6285a7535d..7501c20d8f 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -52,7 +52,7 @@ spec: arm64: cloud-hypervisor: url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "bf004ddc1a148f47caa87ac49a783b8dbd6bf9bc27abe522ed197df7b982d3b1" + sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs); v1.14.0 carries the # vhost-0.16 snapshot/restore fix the kata-bundled v1.13.3 lacks. virtiofsd: @@ -75,7 +75,7 @@ spec: amd64: cloud-hypervisor: url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "829af01ff075bb96c4f183905134c453a88d68cbabdc6b87df21098842581ee9" + sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" # The x86_64-musl binary attached to the upstream v1.14.0 release (downloaded From ed6e70d225e8a78708b48014a5cfe5a31256f1b4 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Fri, 14 Aug 2026 21:16:45 -0700 Subject: [PATCH 2/2] ateom-microvm: boot the kata agent as PID 1 on cloud-hypervisor v53+ The guest ran a full Ubuntu systemd to start one process. Measured on the counter demo, systemd and its library closure were ~40MiB of the ~68MiB the guest read from its rootfs disk at boot; the agent itself links only libc, libm, libgcc_s and ld.so. Boot it directly with init=/usr/bin/kata-agent. The agent already detects PID 1 and does the init work (mounts /proc, /sys, devtmpfs /dev, /dev/shm, /dev/pts, tmpfs /run, cgroups, sets the hostname), which is how kata's own initrd variant runs it. Measured on kind (arm64, counter demo, 1280MiB guest, same base and template, three golden bakes per arm): snapshot 145MiB -> 106.6MiB (-26%) read from /dev/vda 58.6MiB -> 35.0MiB (-40%) guest page cache 63MiB -> 40MiB cold boot 15924ms -> 10308ms (-35%) of which the agent dial 10361ms -> 4675ms (-55%) guest userspace systemd + journald + 2x chronyd + agent + workload -> agent + workload The boot win is the dial phase: the agent is PID 1 rather than a unit systemd reaches several seconds into its startup, so ateom stops waiting on it. It is also much steadier - 10.298/10.310/10.316s across three bakes, against 14.4-17.9s for the same tree with systemd. Gate it on the VMM version, because dropping systemd drops chronyd with it (kata-containers.target wants it) and chronyd is what repaired the guest clock across a resume. cloud-hypervisor v53 advances the guest clock on restore itself; v52 does not, so an older or unreadable version keeps booting systemd. Verified with chronyd masked and 150s of downtime: a v53 guest resumes reading the correct wall clock, a v52 guest is 160s behind, frozen at the pause. The version comes from the vmm.ping LaunchVMM already does, so the gate costs no extra round trip. --- cmd/ateom-microvm/internal/ch/guestclock.go | 36 ++++++++++ .../internal/ch/guestclock_test.go | 50 ++++++++++++++ cmd/ateom-microvm/run.go | 65 ++++++++++++++++--- cmd/ateom-microvm/run_test.go | 24 +++++++ 4 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 cmd/ateom-microvm/internal/ch/guestclock.go create mode 100644 cmd/ateom-microvm/internal/ch/guestclock_test.go diff --git a/cmd/ateom-microvm/internal/ch/guestclock.go b/cmd/ateom-microvm/internal/ch/guestclock.go new file mode 100644 index 0000000000..2036762990 --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/guestclock.go @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +// clockAdvanceSince is the first cloud-hypervisor release that catches the guest +// clock up to wall-clock time when restoring a snapshot: on aarch64 it advances +// CNTVCT_EL0 by the downtime, and on x86 it keeps KVM_CLOCK_REALTIME set so the +// kernel adjusts kvmclock at KVM_SET_CLOCK. Before it, a restored guest resumes +// frozen at the instant it was snapshotted and something in the guest has to notice. +var clockAdvanceSince = [3]int{53, 0, 0} + +// AdvancesGuestClockOnRestore reports whether this VMM repairs the guest clock +// across a restore by itself. +// +// It reports false when the version cannot be read or parsed, which is the safe +// direction: a caller that wrongly believes the VMM corrects the clock leaves the +// guest reading a stale time after every resume, with no error to show for it. +func (i VMMInfo) AdvancesGuestClockOnRestore() bool { + v, ok := i.semver() + if !ok { + return false + } + return compareVersions(v, clockAdvanceSince) >= 0 +} diff --git a/cmd/ateom-microvm/internal/ch/guestclock_test.go b/cmd/ateom-microvm/internal/ch/guestclock_test.go new file mode 100644 index 0000000000..dc984e11bd --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/guestclock_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +import "testing" + +func TestAdvancesGuestClockOnRestore(t *testing.T) { + for _, tc := range []struct { + name string + info VMMInfo + want bool + }{ + // Real payloads, as reported by the binaries we ship. Measured: with chronyd + // masked and 150s of downtime, a v53 guest resumes reading the correct wall + // clock and a v52 guest is 160s behind. + {"v52 does not", VMMInfo{Version: "52.0.0", BuildVersion: "v52.0"}, false}, + {"v53 does", VMMInfo{Version: "53.0.0", BuildVersion: "v53.0"}, true}, + {"later releases keep the fix", VMMInfo{Version: "60.1.2"}, true}, + {"much older", VMMInfo{Version: "41.0.0"}, false}, + + // The semver field is preferred, but the release tag is enough on its own. + {"tag only", VMMInfo{BuildVersion: "v52.0"}, false}, + {"tag only, fixed", VMMInfo{BuildVersion: "v53.0"}, true}, + {"suffixed", VMMInfo{Version: "53.0.0-dirty"}, true}, + + // Unknown means "assume it does not": believing a VMM corrects the clock when + // it does not leaves every resumed guest reading a stale time, silently. + {"empty", VMMInfo{}, false}, + {"garbage", VMMInfo{Version: "not-a-version"}, false}, + {"partial garbage", VMMInfo{Version: "53.x.0"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.info.AdvancesGuestClockOnRestore(); got != tc.want { + t.Errorf("AdvancesGuestClockOnRestore() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 3e4b676c3b..b4abd43f64 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -117,6 +117,10 @@ const ( assetVirtiofsd = "virtiofsd" ) +// kataAgentPath is where the kata guest image keeps the agent binary. buildVMConfig +// boots it as PID 1 (init=), so it is the guest's entire userspace. +const kataAgentPath = "/usr/bin/kata-agent" + // vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit // for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same // pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per @@ -481,7 +485,8 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // host-side overlay upper through the shared mount). serialLog is also read on a // failed agent dial below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable) + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable, + agentInit(ctx, client.Info())) if err := client.CreateVM(ctx, vmCfg); err != nil { return fmt.Errorf("while creating VM: %w", err) } @@ -725,24 +730,66 @@ func (s *AteomService) guestSize(sz sizing.SandboxSize) (sizing.SandboxSize, err return sz, nil } -// buildVMConfig assembles the cloud-hypervisor VmConfig. The kernel cmdline replicates -// kata's clh boot cmdline; beyond the base params it must set -// systemd.unit=kata-containers.target (else the guest powers off ~6s in) and mask -// systemd-networkd (the agent owns eth0). The console is arch-specific: ttyAMA0 on -// arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO lower is -// the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no actor disks. +// agentInit reports whether to boot the kata agent as the guest's PID 1, from what +// the VMM just told us about itself over vmm.ping. +// +// Booting the agent directly skips systemd entirely, which is most of what the guest +// reads at boot and most of what its snapshot carries. The catch is that it also drops +// chronyd (kata-containers.target wants it), and chronyd is what repairs the guest +// clock after a resume — so this is only safe on a VMM that advances the guest clock +// across a restore itself. On an older or unreadable version, boot systemd instead and +// keep the guest correct at the cost of the memory. +func agentInit(ctx context.Context, info ch.VMMInfo) bool { + if info.AdvancesGuestClockOnRestore() { + return true + } + slog.InfoContext(ctx, "VMM does not advance the guest clock on restore; booting systemd to keep chronyd", + slog.String("vmm_version", info.Version), slog.String("vmm_build_version", info.BuildVersion)) + return false +} + +// initParams returns the kernel cmdline parameters that select the guest's PID 1. +// The systemd path must name kata's target (else the guest powers off ~6s in) and +// mask systemd-networkd, since the agent owns eth0. +func initParams(agentInit bool) string { + if agentInit { + return "init=" + kataAgentPath + } + return "systemd.unit=kata-containers.target " + + "systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket" +} + +// buildVMConfig assembles the cloud-hypervisor VmConfig. The console is arch-specific: +// ttyAMA0 on arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO +// lower is the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no +// actor disks. +// +// init=kataAgentPath boots the kata agent as PID 1 instead of systemd. The agent detects +// that it is PID 1 and does the init work itself: it mounts /proc, /sys, devtmpfs /dev, +// /dev/shm, /dev/pts, tmpfs /run and the cgroup hierarchy, then serves ttrpc over vsock. +// Nothing else in the guest image is ours to run — the workload is a container the agent +// starts — so systemd only cost us. Measured on the counter demo, dropping it took the +// guest's boot-time reads from this disk from 58.6MiB to 35.0MiB, the snapshot from 145MiB +// to 106.6MiB at the same guest RAM, and a cold boot from 15.9s to 10.3s: the agent is +// PID 1 rather than a unit systemd reaches several seconds in, so ateom stops waiting for +// it (the dial phase goes 10.4s -> 4.7s). +// +// Dropping systemd also drops chronyd (kata-containers.target wants it), which is what +// used to repair the guest clock after a resume. That is safe only from cloud-hypervisor +// v53, which advances the guest clock across a restore itself; on v52 a restored guest +// stays frozen at the instant it was snapshotted. // // withDurable adds a second virtio-fs device for the actor's writable durable-dir // volumes (see durable.go), served by its own virtiofsd on the same PCI segment. // The disk-backed rootfs upper share (see rootfsupper.go) is always present. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable, agentInit bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" } cmdline := "root=/dev/vda1 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 " + "panic=1 no_timer_check noreplace-smp console=" + console + ",115200n8 " + - "systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket" + initParams(agentInit) if kparams != "" { cmdline += " " + kparams } diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index fca582b1ea..5d648cb188 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -21,6 +21,7 @@ import ( "errors" "net" "path/filepath" + "strings" "testing" "time" @@ -174,3 +175,26 @@ func TestGuestSize(t *testing.T) { t.Errorf("guestSize(%dMiB) = %+v, nil; want an error", reserve+1, gotErr) } } + +func TestInitParams(t *testing.T) { + // The agent path must be the one the kata guest image actually ships, since the + // kernel silently panics on an init= that does not exist. + if got := initParams(true); got != "init=/usr/bin/kata-agent" { + t.Errorf("initParams(true) = %q", got) + } + // Without the agent as PID 1, systemd needs kata's target — it powers the guest + // off within seconds otherwise — and networkd must stay masked, the agent owns eth0. + systemd := initParams(false) + for _, want := range []string{ + "systemd.unit=kata-containers.target", + "systemd.mask=systemd-networkd.service", + "systemd.mask=systemd-networkd.socket", + } { + if !strings.Contains(systemd, want) { + t.Errorf("initParams(false) = %q, missing %q", systemd, want) + } + } + if strings.Contains(systemd, "init=") { + t.Errorf("initParams(false) = %q, must not override init", systemd) + } +}