diff --git a/cmd/ateom-microvm/internal/ch/guestclock.go b/cmd/ateom-microvm/internal/ch/guestclock.go new file mode 100644 index 0000000000..2036762990 --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/guestclock.go @@ -0,0 +1,36 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +// clockAdvanceSince is the first cloud-hypervisor release that catches the guest +// clock up to wall-clock time when restoring a snapshot: on aarch64 it advances +// CNTVCT_EL0 by the downtime, and on x86 it keeps KVM_CLOCK_REALTIME set so the +// kernel adjusts kvmclock at KVM_SET_CLOCK. Before it, a restored guest resumes +// frozen at the instant it was snapshotted and something in the guest has to notice. +var clockAdvanceSince = [3]int{53, 0, 0} + +// AdvancesGuestClockOnRestore reports whether this VMM repairs the guest clock +// across a restore by itself. +// +// It reports false when the version cannot be read or parsed, which is the safe +// direction: a caller that wrongly believes the VMM corrects the clock leaves the +// guest reading a stale time after every resume, with no error to show for it. +func (i VMMInfo) AdvancesGuestClockOnRestore() bool { + v, ok := i.semver() + if !ok { + return false + } + return compareVersions(v, clockAdvanceSince) >= 0 +} diff --git a/cmd/ateom-microvm/internal/ch/guestclock_test.go b/cmd/ateom-microvm/internal/ch/guestclock_test.go new file mode 100644 index 0000000000..dc984e11bd --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/guestclock_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +import "testing" + +func TestAdvancesGuestClockOnRestore(t *testing.T) { + for _, tc := range []struct { + name string + info VMMInfo + want bool + }{ + // Real payloads, as reported by the binaries we ship. Measured: with chronyd + // masked and 150s of downtime, a v53 guest resumes reading the correct wall + // clock and a v52 guest is 160s behind. + {"v52 does not", VMMInfo{Version: "52.0.0", BuildVersion: "v52.0"}, false}, + {"v53 does", VMMInfo{Version: "53.0.0", BuildVersion: "v53.0"}, true}, + {"later releases keep the fix", VMMInfo{Version: "60.1.2"}, true}, + {"much older", VMMInfo{Version: "41.0.0"}, false}, + + // The semver field is preferred, but the release tag is enough on its own. + {"tag only", VMMInfo{BuildVersion: "v52.0"}, false}, + {"tag only, fixed", VMMInfo{BuildVersion: "v53.0"}, true}, + {"suffixed", VMMInfo{Version: "53.0.0-dirty"}, true}, + + // Unknown means "assume it does not": believing a VMM corrects the clock when + // it does not leaves every resumed guest reading a stale time, silently. + {"empty", VMMInfo{}, false}, + {"garbage", VMMInfo{Version: "not-a-version"}, false}, + {"partial garbage", VMMInfo{Version: "53.x.0"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.info.AdvancesGuestClockOnRestore(); got != tc.want { + t.Errorf("AdvancesGuestClockOnRestore() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 3e4b676c3b..b4abd43f64 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -117,6 +117,10 @@ const ( assetVirtiofsd = "virtiofsd" ) +// kataAgentPath is where the kata guest image keeps the agent binary. buildVMConfig +// boots it as PID 1 (init=), so it is the guest's entire userspace. +const kataAgentPath = "/usr/bin/kata-agent" + // vmmMemReserveMiB is the DEFAULT guest RAM held back from the pod's memory limit // for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the same // pod cgroup as the guest RAM; without a margin the pod OOMs. Overridable per @@ -481,7 +485,8 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // host-side overlay upper through the shared mount). serialLog is also read on a // failed agent dial below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable) + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable, + agentInit(ctx, client.Info())) if err := client.CreateVM(ctx, vmCfg); err != nil { return fmt.Errorf("while creating VM: %w", err) } @@ -725,24 +730,66 @@ func (s *AteomService) guestSize(sz sizing.SandboxSize) (sizing.SandboxSize, err return sz, nil } -// buildVMConfig assembles the cloud-hypervisor VmConfig. The kernel cmdline replicates -// kata's clh boot cmdline; beyond the base params it must set -// systemd.unit=kata-containers.target (else the guest powers off ~6s in) and mask -// systemd-networkd (the agent owns eth0). The console is arch-specific: ttyAMA0 on -// arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO lower is -// the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no actor disks. +// agentInit reports whether to boot the kata agent as the guest's PID 1, from what +// the VMM just told us about itself over vmm.ping. +// +// Booting the agent directly skips systemd entirely, which is most of what the guest +// reads at boot and most of what its snapshot carries. The catch is that it also drops +// chronyd (kata-containers.target wants it), and chronyd is what repairs the guest +// clock after a resume — so this is only safe on a VMM that advances the guest clock +// across a restore itself. On an older or unreadable version, boot systemd instead and +// keep the guest correct at the cost of the memory. +func agentInit(ctx context.Context, info ch.VMMInfo) bool { + if info.AdvancesGuestClockOnRestore() { + return true + } + slog.InfoContext(ctx, "VMM does not advance the guest clock on restore; booting systemd to keep chronyd", + slog.String("vmm_version", info.Version), slog.String("vmm_build_version", info.BuildVersion)) + return false +} + +// initParams returns the kernel cmdline parameters that select the guest's PID 1. +// The systemd path must name kata's target (else the guest powers off ~6s in) and +// mask systemd-networkd, since the agent owns eth0. +func initParams(agentInit bool) string { + if agentInit { + return "init=" + kataAgentPath + } + return "systemd.unit=kata-containers.target " + + "systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket" +} + +// buildVMConfig assembles the cloud-hypervisor VmConfig. The console is arch-specific: +// ttyAMA0 on arm64, ttyS0 on amd64. /dev/vda is the RO guest image; the actor rootfs's RO +// lower is the virtio-fs device on PCI segment 1 (hence num_pci_segments=2), with no +// actor disks. +// +// init=kataAgentPath boots the kata agent as PID 1 instead of systemd. The agent detects +// that it is PID 1 and does the init work itself: it mounts /proc, /sys, devtmpfs /dev, +// /dev/shm, /dev/pts, tmpfs /run and the cgroup hierarchy, then serves ttrpc over vsock. +// Nothing else in the guest image is ours to run — the workload is a container the agent +// starts — so systemd only cost us. Measured on the counter demo, dropping it took the +// guest's boot-time reads from this disk from 58.6MiB to 35.0MiB, the snapshot from 145MiB +// to 106.6MiB at the same guest RAM, and a cold boot from 15.9s to 10.3s: the agent is +// PID 1 rather than a unit systemd reaches several seconds in, so ateom stops waiting for +// it (the dial phase goes 10.4s -> 4.7s). +// +// Dropping systemd also drops chronyd (kata-containers.target wants it), which is what +// used to repair the guest clock after a resume. That is safe only from cloud-hypervisor +// v53, which advances the guest clock across a restore itself; on v52 a restored guest +// stays frozen at the instant it was snapshotted. // // withDurable adds a second virtio-fs device for the actor's writable durable-dir // volumes (see durable.go), served by its own virtiofsd on the same PCI segment. // The disk-backed rootfs upper share (see rootfsupper.go) is always present. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable, agentInit bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" } cmdline := "root=/dev/vda1 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 " + "panic=1 no_timer_check noreplace-smp console=" + console + ",115200n8 " + - "systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service systemd.mask=systemd-networkd.socket" + initParams(agentInit) if kparams != "" { cmdline += " " + kparams } diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index fca582b1ea..5d648cb188 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -21,6 +21,7 @@ import ( "errors" "net" "path/filepath" + "strings" "testing" "time" @@ -174,3 +175,26 @@ func TestGuestSize(t *testing.T) { t.Errorf("guestSize(%dMiB) = %+v, nil; want an error", reserve+1, gotErr) } } + +func TestInitParams(t *testing.T) { + // The agent path must be the one the kata guest image actually ships, since the + // kernel silently panics on an init= that does not exist. + if got := initParams(true); got != "init=/usr/bin/kata-agent" { + t.Errorf("initParams(true) = %q", got) + } + // Without the agent as PID 1, systemd needs kata's target — it powers the guest + // off within seconds otherwise — and networkd must stay masked, the agent owns eth0. + systemd := initParams(false) + for _, want := range []string{ + "systemd.unit=kata-containers.target", + "systemd.mask=systemd-networkd.service", + "systemd.mask=systemd-networkd.socket", + } { + if !strings.Contains(systemd, want) { + t.Errorf("initParams(false) = %q, missing %q", systemd, want) + } + } + if strings.Contains(systemd, "init=") { + t.Errorf("initParams(false) = %q, must not override init", systemd) + } +} diff --git a/hack/microvm-assets/assemble.sh b/hack/microvm-assets/assemble.sh index ab0c3e7a14..cb01d9ae63 100755 --- a/hack/microvm-assets/assemble.sh +++ b/hack/microvm-assets/assemble.sh @@ -34,7 +34,7 @@ # publishes a prebuilt static binary for x86_64 only; arm64 builds from the release # tag, which needs rust (rustup) + libcap-ng-dev libseccomp-dev pkg-config. # -# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.0.0), CH_VER (v52.0), +# Env: ARCH (arm64|amd64, default arm64), KATA_VER (4.0.0), CH_VER (v53.0), # OUT (default ./bin/microvm-assets/$ARCH, under the gitignored bin/). set -o errexit -o nounset -o pipefail @@ -43,7 +43,7 @@ ROOT="$(git rev-parse --show-toplevel)" ARCH="${ARCH:-arm64}" KATA_VER="${KATA_VER:-4.0.0}" -CH_VER="${CH_VER:-v52.0}" +CH_VER="${CH_VER:-v53.0}" OUT="${OUT:-${ROOT}/bin/microvm-assets/$ARCH}" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index 6285a7535d..7501c20d8f 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -52,7 +52,7 @@ spec: arm64: cloud-hypervisor: url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "bf004ddc1a148f47caa87ac49a783b8dbd6bf9bc27abe522ed197df7b982d3b1" + sha256: "f192b510eea1c710cbc439d716bb0573c223fc463dbe3e6523788a2b7ef62850" # virtiofsd serves the overlay RO lower (virtio-fs); v1.14.0 carries the # vhost-0.16 snapshot/restore fix the kata-bundled v1.13.3 lacks. virtiofsd: @@ -75,7 +75,7 @@ spec: amd64: cloud-hypervisor: url: "gs://${BUCKET_NAME}/kata-assets/cloud-hypervisor" - sha256: "829af01ff075bb96c4f183905134c453a88d68cbabdc6b87df21098842581ee9" + sha256: "448af3d4e59b22c2987f7df94c213ad40fb53a10d437e42b5ee6c4fce7c29ecc" virtiofsd: url: "gs://${BUCKET_NAME}/kata-assets/virtiofsd" # The x86_64-musl binary attached to the upstream v1.14.0 release (downloaded