diff --git a/CHANGELOG.md b/CHANGELOG.md index 00594206..06684bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,42 @@ snapshot is fully published, so a tag always holds a consistent src == dst guard comment duplication and makes the chain-unpack tail bail instead of resolving an empty path. +### Restored sandboxes get a per-child writable layer in guest RAM + +A snapshot's rootfs is a single ext4 file. Firecracker freezes the drive's +path and read-only flag into the vmstate and cannot re-point either at +restore, so every child restored from a tag opened the same ext4 — and when +a bake opened it read-write, concurrent children were separate guest kernels +writing one filesystem with no coordinator. That corrupts package files and +directory entries at random (`/usr/bin/uname` holding another file's bytes, +`EBADMSG` on `/var/lib/dpkg` entries, `Exec format error`) and surfaces as a +random build failure rather than a sandbox error. + +Bakes now open the rootfs read-only and the guest kernel mounts it `ro` +(`BootConfig::ext4_overlay`), so the shared base is provably never written. +`/forkd-init.sh` supplies the writable layer in guest RAM: tmpfs for `/run`, +`/dev/shm` and `/tmp`, and an overlayfs — image content as the lower layer, +tmpfs upper — for `/etc`, `/root`, `/home`, `/opt`, `/srv`, `/usr/local` and +`/var`. Because that state is guest memory, `memory.bin` carries it into +every child, so it survives a BRANCH; that is the same property the guest's +`/tmp` tmpfs has always relied on. Keeping the image content as the lower +layer rather than masking it with a bare tmpfs is what preserves preinstalled +toolchains and the caches warmed during the bake. + +Two consequences worth planning for: + +- **Guest RAM now bounds writable space.** The writable tmpfs is capped + (2 GiB by default, `forkd.rw_size=` on the kernel cmdline via + `BootConfig::with_rw_size`) so a job that outgrows it fails its own write + with ENOSPC instead of OOMing the guest. Size `mem_size_mib` to cover the + job's writable footprint. +- **A write outside the provided set fails with EROFS** instead of mutating + the shared base. Paths the image does not have are skipped, since a + read-only root cannot be mkdir'd into. + +Tags baked before this keep the read-write drive their vmstate froze; re-bake +to pick the layer up. + ### Rootfs sidecar placement: recorded absolute path, validated Packs record the rootfs sidecar's target as the vmstate-frozen ABSOLUTE diff --git a/crates/forkd-cli/src/main.rs b/crates/forkd-cli/src/main.rs index 58232022..ab08360a 100644 --- a/crates/forkd-cli/src/main.rs +++ b/crates/forkd-cli/src/main.rs @@ -2958,7 +2958,15 @@ fn snapshot_cmd( let work_dir = std::env::temp_dir().join(format!("forkd-parent-{tag}")); let mut cfg = if rw { - BootConfig::ext4_rw(kernel, boot_rootfs, work_dir.clone()) + // The parent boots the ext4 READ-ONLY and takes its writable layer + // in guest RAM (see BootConfig::ext4_overlay). The rootfs is one + // file shared by every child restored from this snapshot, so the + // guest must never write it — two children writing one ext4 with + // no coordinator is how they corrupt each other. Warmup writes + // land in the tmpfs/overlay upper, and memory.bin carries them + // into every child, which is the same reason the /tmp tmpfs has + // always survived a BRANCH. + BootConfig::ext4_overlay(kernel, boot_rootfs, work_dir.clone()) } else { BootConfig::quickstart(kernel, boot_rootfs, work_dir.clone()) }; diff --git a/crates/forkd-controller/src/http.rs b/crates/forkd-controller/src/http.rs index 1b559995..881c6d1c 100644 --- a/crates/forkd-controller/src/http.rs +++ b/crates/forkd-controller/src/http.rs @@ -1084,7 +1084,10 @@ fn build_snapshot_boot_config( .map(|e| e.eq_ignore_ascii_case("ext4")) .unwrap_or(false); let mut cfg = if rootfs_ext4 { - BootConfig::ext4_rw( + // Read-only ext4 with the writable layer in guest RAM: the rootfs + // is shared by every child restored from this snapshot, so the + // guest must never write it (see BootConfig::ext4_overlay). + BootConfig::ext4_overlay( kernel.to_path_buf(), rootfs.to_path_buf(), work_dir.to_path_buf(), diff --git a/crates/forkd-vmm/src/lib.rs b/crates/forkd-vmm/src/lib.rs index e94a3829..567d2e8c 100644 --- a/crates/forkd-vmm/src/lib.rs +++ b/crates/forkd-vmm/src/lib.rs @@ -182,6 +182,56 @@ impl BootConfig { } } + /// Boot config for a **read-only ext4 rootfs with an in-guest writable + /// layer** — the right shape for any snapshot that will be restored + /// more than once. + /// + /// The ext4 is opened read-only on the host *and* mounted `ro` by the + /// guest kernel, so nothing can write it; `/forkd-init.sh` provides + /// tmpfs and overlayfs mounts in guest RAM for everything writable. + /// That is what makes concurrent children safe: they share a base that + /// is provably never written, instead of one ext4 opened read-write by + /// every child. Firecracker freezes the drive's path and read-only flag + /// in the vmstate and cannot re-point them at restore, so "never + /// written" is the only property that can be enforced from the host. + /// + /// Writable state lives in guest memory, which `memory.bin` captures, + /// so it survives a BRANCH — the same reason the guest's `/tmp` tmpfs + /// always has. + /// + /// Guest RAM has to cover the writable footprint: the tmpfs is capped + /// (2 GiB in the guest unless `with_rw_size` overrides it), so a job + /// that outgrows it fails its own write with ENOSPC instead of + /// silently corrupting the shared base or OOMing the guest. + pub fn ext4_overlay(kernel: PathBuf, rootfs: PathBuf, work_dir: PathBuf) -> Self { + Self { + kernel, + rootfs, + vcpu_count: 2, + mem_size_mib: 512, + // `ro` is load-bearing, not cosmetic: it is what keeps the + // shared ext4 byte-identical across every restored child. + // clocksource/trust_cpu carry the same rationale as + // quickstart(); `init=` is required because a read-only root + // cannot let an image's own init run. + boot_args: + "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda ro init=/forkd-init.sh random.trust_cpu=on clocksource=kvm-clock".into(), + work_dir, + rootfs_read_only: true, + network: None, + volumes: Vec::new(), + } + } + + /// Size cap for the guest's writable layer (overlay uppers plus + /// scratch), appended as a `forkd.rw_size=` kernel cmdline hint that + /// `/forkd-init.sh` reads. Takes a tmpfs size string, e.g. `"8g"`. + /// Defaults to 2 GiB in the guest. + pub fn with_rw_size(mut self, size: &str) -> Self { + self.boot_args.push_str(&format!(" forkd.rw_size={size}")); + self + } + /// Attach a virtio-net interface to this VM. The tap device referenced /// by `host_dev_name` must already exist on the host. pub fn with_network(mut self, net: NetworkConfig) -> Self { @@ -2836,6 +2886,60 @@ mod tests { ); } + /// The overlay config is what keeps a restored snapshot's rootfs + /// byte-identical: the drive must be opened read-only on the host AND + /// mounted `ro` in the guest, and the guest must run our init so it can + /// provide the writable layer. Getting this wrong is silent corruption + /// rather than a failing test in production, so assert all three. + #[test] + fn boot_config_ext4_overlay_is_read_only_with_guest_writable_layer() { + let cfg = BootConfig::ext4_overlay("/tmp/k".into(), "/tmp/r.ext4".into(), "/tmp/w".into()); + assert!( + cfg.rootfs_read_only, + "the host must open the shared ext4 read-only" + ); + assert!( + cfg.boot_args.contains(" root=/dev/vda ro "), + "the guest kernel must mount the shared ext4 read-only, got: {}", + cfg.boot_args + ); + assert!( + !cfg.boot_args.contains(" rw "), + "ext4_overlay must not request a writable root: `{}`", + cfg.boot_args + ); + assert!( + cfg.boot_args.contains("init=/forkd-init.sh"), + "the guest writable layer lives in /forkd-init.sh: `{}`", + cfg.boot_args + ); + // Same rationale as quickstart(); a later token would override it. + assert!(cfg.boot_args.contains("clocksource=kvm-clock")); + let after = cfg + .boot_args + .split("clocksource=kvm-clock") + .nth(1) + .unwrap_or(""); + assert!( + !after.contains("clocksource="), + "duplicate clocksource token" + ); + } + + /// The writable layer is capped, and the cap is reachable from the host + /// because a job that outgrows it must fail its own write rather than + /// OOM the guest. + #[test] + fn with_rw_size_appends_the_cmdline_hint() { + let cfg = BootConfig::ext4_overlay("/tmp/k".into(), "/tmp/r.ext4".into(), "/tmp/w".into()) + .with_rw_size("8g"); + assert!( + cfg.boot_args.contains("forkd.rw_size=8g"), + "init reads the cap from the cmdline: `{}`", + cfg.boot_args + ); + } + #[test] fn volume_device_name_progression() { assert_eq!(volume_device_name(0).unwrap(), "vdb"); diff --git a/rootfs-init/forkd-init.sh b/rootfs-init/forkd-init.sh index 44e9cc66..7920e772 100644 --- a/rootfs-init/forkd-init.sh +++ b/rootfs-init/forkd-init.sh @@ -6,15 +6,63 @@ mount -t proc proc /proc 2>/dev/null mount -t sysfs sys /sys 2>/dev/null mount -t devtmpfs devtmpfs /dev 2>/dev/null -# Mount /tmp as tmpfs so per-sandbox scratch writes (agent logs, -# socket paths, hint files used by demos) land in this VM's memory -# instead of the shared on-disk rootfs.ext4. Without this, multiple -# sandboxes booted from the same snapshot file would concurrently -# write to the same inode in the same loop-mounted ext4 and corrupt -# it ("Structure needs cleaning" on the next access). -# tmpfs state survives BRANCH because it lives in guest RAM, which -# is what memory.bin captures. -mount -t tmpfs -o size=256m tmpfs /tmp 2>/dev/null +# --- writable layer --------------------------------------------------- +# The rootfs is booted READ-ONLY (root=/dev/vda ro) and is one ext4 file +# shared by every sandbox restored from this snapshot, so nothing may +# write it: two guests writing one filesystem with no coordinator corrupt +# it ("Structure needs cleaning", foreign bytes inside package files). +# Everything writable is therefore provided here, in guest RAM. +# +# RAM is the point, not a limitation: tmpfs and overlayfs-upper pages are +# guest memory, and memory.bin is what a snapshot captures, so writable +# state survives a BRANCH exactly as the /tmp tmpfs always has. +# +# plain tmpfs /run /dev/shm -- scratch that owns nothing +# overlay /etc /root /home -- the image's baked content stays +# /opt /srv visible as the lower layer, writes +# /usr/local /var land in the tmpfs upper +# +# The overlay list is where jobs actually write (cargo registry and +# target, pnpm store, mix _build, the checkout itself) while keeping the +# image's preinstalled content — a bare tmpfs over those paths would hide +# it. /etc is on the list because the DNS fix below writes resolv.conf. +# +# A path that does not exist in the image is skipped, since a read-only +# root cannot be mkdir'd into. A write outside this set fails with EROFS: +# loud, and far better than silently mutating the shared base. +FORKD_RW_SIZE="$(grep -oE 'forkd\.rw_size=[^ ]+' /proc/cmdline | head -1 | cut -d= -f2-)" +[ -n "$FORKD_RW_SIZE" ] || FORKD_RW_SIZE=2g +if mount -t tmpfs -o "size=$FORKD_RW_SIZE" tmpfs /tmp 2>/dev/null; then + echo "forkd-init: writable layer $FORKD_RW_SIZE (tmpfs at /tmp)" +else + echo "forkd-init: WARN no writable tmpfs at /tmp; sandboxes will be read-only" >&2 +fi +# Overlay upper/work dirs live on that tmpfs: overlayfs needs them on the +# same filesystem, and /tmp is the one path we know is writable. +mkdir -p /tmp/.rw/upper /tmp/.rw/work 2>/dev/null + +_forkd_tmpfs() { # + [ -d "$1" ] || return 0 + mount -t tmpfs -o "size=$FORKD_RW_SIZE" tmpfs "$1" 2>/dev/null || \ + echo "forkd-init: WARN $1 not writable (tmpfs mount failed)" >&2 +} + +_forkd_overlay() { # + [ -d "$1" ] || return 0 + install -d "/tmp/.rw/upper$1" "/tmp/.rw/work$1" 2>/dev/null + if mount -t overlay overlay \ + -o "lowerdir=$1,upperdir=/tmp/.rw/upper$1,workdir=/tmp/.rw/work$1" "$1" 2>/dev/null; then + echo "forkd-init: $1 writable (overlay upper in RAM)" + else + echo "forkd-init: WARN $1 left read-only (overlay mount failed)" >&2 + fi +} + +_forkd_tmpfs /run +_forkd_tmpfs /dev/shm +for _p in /etc /root /home /opt /srv /usr/local /var; do + _forkd_overlay "$_p" +done # Make sure PATH covers both Ubuntu (/usr/bin) and official python (/usr/local/bin) # images. Subprocess invocations from the agent inherit this.