feat(vmm,init): per-child writable layer in guest RAM; bakes boot the rootfs read-only

CHANGELOG.md

@@ -33,6 +33,42 @@ snapshot is fully published, so a tag always holds a consistent
 src == dst guard comment duplication and makes the chain-unpack tail
 bail instead of resolving an empty path.
 
+### Restored sandboxes get a per-child writable layer in guest RAM
+
+A snapshot's rootfs is a single ext4 file. Firecracker freezes the drive's
+path and read-only flag into the vmstate and cannot re-point either at
+restore, so every child restored from a tag opened the same ext4 — and when
+a bake opened it read-write, concurrent children were separate guest kernels
+writing one filesystem with no coordinator. That corrupts package files and
+directory entries at random (`/usr/bin/uname` holding another file's bytes,
+`EBADMSG` on `/var/lib/dpkg` entries, `Exec format error`) and surfaces as a
+random build failure rather than a sandbox error.
+
+Bakes now open the rootfs read-only and the guest kernel mounts it `ro`
+(`BootConfig::ext4_overlay`), so the shared base is provably never written.
+`/forkd-init.sh` supplies the writable layer in guest RAM: tmpfs for `/run`,
+`/dev/shm` and `/tmp`, and an overlayfs — image content as the lower layer,
+tmpfs upper — for `/etc`, `/root`, `/home`, `/opt`, `/srv`, `/usr/local` and
+`/var`. Because that state is guest memory, `memory.bin` carries it into
+every child, so it survives a BRANCH; that is the same property the guest's
+`/tmp` tmpfs has always relied on. Keeping the image content as the lower
+layer rather than masking it with a bare tmpfs is what preserves preinstalled
+toolchains and the caches warmed during the bake.
+
+Two consequences worth planning for:
+
+- **Guest RAM now bounds writable space.** The writable tmpfs is capped
+  (2 GiB by default, `forkd.rw_size=<size>` on the kernel cmdline via
+  `BootConfig::with_rw_size`) so a job that outgrows it fails its own write
+  with ENOSPC instead of OOMing the guest. Size `mem_size_mib` to cover the
+  job's writable footprint.
+- **A write outside the provided set fails with EROFS** instead of mutating
+  the shared base. Paths the image does not have are skipped, since a
+  read-only root cannot be mkdir'd into.
+
+Tags baked before this keep the read-write drive their vmstate froze; re-bake
+to pick the layer up.
+
 ### Rootfs sidecar placement: recorded absolute path, validated
 
 Packs record the rootfs sidecar's target as the vmstate-frozen ABSOLUTE

crates/forkd-cli/src/main.rs

@@ -2958,7 +2958,15 @@ fn snapshot_cmd(
 
     let work_dir = std::env::temp_dir().join(format!("forkd-parent-{tag}"));
     let mut cfg = if rw {
-        BootConfig::ext4_rw(kernel, boot_rootfs, work_dir.clone())
+        // The parent boots the ext4 READ-ONLY and takes its writable layer
+        // in guest RAM (see BootConfig::ext4_overlay). The rootfs is one
+        // file shared by every child restored from this snapshot, so the
+        // guest must never write it — two children writing one ext4 with
+        // no coordinator is how they corrupt each other. Warmup writes
+        // land in the tmpfs/overlay upper, and memory.bin carries them
+        // into every child, which is the same reason the /tmp tmpfs has
+        // always survived a BRANCH.
+        BootConfig::ext4_overlay(kernel, boot_rootfs, work_dir.clone())
     } else {
         BootConfig::quickstart(kernel, boot_rootfs, work_dir.clone())
     };

crates/forkd-controller/src/http.rs

@@ -1084,7 +1084,10 @@ fn build_snapshot_boot_config(
             .map(|e| e.eq_ignore_ascii_case("ext4"))
             .unwrap_or(false);
     let mut cfg = if rootfs_ext4 {
-        BootConfig::ext4_rw(
+        // Read-only ext4 with the writable layer in guest RAM: the rootfs
+        // is shared by every child restored from this snapshot, so the
+        // guest must never write it (see BootConfig::ext4_overlay).
+        BootConfig::ext4_overlay(
             kernel.to_path_buf(),
             rootfs.to_path_buf(),
             work_dir.to_path_buf(),

crates/forkd-vmm/src/lib.rs

@@ -182,6 +182,56 @@ impl BootConfig {
         }
     }
 
+    /// Boot config for a **read-only ext4 rootfs with an in-guest writable
+    /// layer** — the right shape for any snapshot that will be restored
+    /// more than once.
+    ///
+    /// The ext4 is opened read-only on the host *and* mounted `ro` by the
+    /// guest kernel, so nothing can write it; `/forkd-init.sh` provides
+    /// tmpfs and overlayfs mounts in guest RAM for everything writable.
+    /// That is what makes concurrent children safe: they share a base that
+    /// is provably never written, instead of one ext4 opened read-write by
+    /// every child. Firecracker freezes the drive's path and read-only flag
+    /// in the vmstate and cannot re-point them at restore, so "never
+    /// written" is the only property that can be enforced from the host.
+    ///
+    /// Writable state lives in guest memory, which `memory.bin` captures,
+    /// so it survives a BRANCH — the same reason the guest's `/tmp` tmpfs
+    /// always has.
+    ///
+    /// Guest RAM has to cover the writable footprint: the tmpfs is capped
+    /// (2 GiB in the guest unless `with_rw_size` overrides it), so a job
+    /// that outgrows it fails its own write with ENOSPC instead of
+    /// silently corrupting the shared base or OOMing the guest.
+    pub fn ext4_overlay(kernel: PathBuf, rootfs: PathBuf, work_dir: PathBuf) -> Self {
+        Self {
+            kernel,
+            rootfs,
+            vcpu_count: 2,
+            mem_size_mib: 512,
+            // `ro` is load-bearing, not cosmetic: it is what keeps the
+            // shared ext4 byte-identical across every restored child.
+            // clocksource/trust_cpu carry the same rationale as
+            // quickstart(); `init=` is required because a read-only root
+            // cannot let an image's own init run.
+            boot_args:
+                "console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda ro init=/forkd-init.sh random.trust_cpu=on clocksource=kvm-clock".into(),
+            work_dir,
+            rootfs_read_only: true,
+            network: None,
+            volumes: Vec::new(),
+        }
+    }
+
+    /// Size cap for the guest's writable layer (overlay uppers plus
+    /// scratch), appended as a `forkd.rw_size=` kernel cmdline hint that
+    /// `/forkd-init.sh` reads. Takes a tmpfs size string, e.g. `"8g"`.
+    /// Defaults to 2 GiB in the guest.
+    pub fn with_rw_size(mut self, size: &str) -> Self {
+        self.boot_args.push_str(&format!(" forkd.rw_size={size}"));
+        self
+    }
+
     /// Attach a virtio-net interface to this VM. The tap device referenced
     /// by `host_dev_name` must already exist on the host.
     pub fn with_network(mut self, net: NetworkConfig) -> Self {
@@ -2836,6 +2886,60 @@ mod tests {
         );
     }
 
+    /// The overlay config is what keeps a restored snapshot's rootfs
+    /// byte-identical: the drive must be opened read-only on the host AND
+    /// mounted `ro` in the guest, and the guest must run our init so it can
+    /// provide the writable layer. Getting this wrong is silent corruption
+    /// rather than a failing test in production, so assert all three.
+    #[test]
+    fn boot_config_ext4_overlay_is_read_only_with_guest_writable_layer() {
+        let cfg = BootConfig::ext4_overlay("/tmp/k".into(), "/tmp/r.ext4".into(), "/tmp/w".into());
+        assert!(
+            cfg.rootfs_read_only,
+            "the host must open the shared ext4 read-only"
+        );
+        assert!(
+            cfg.boot_args.contains(" root=/dev/vda ro "),
+            "the guest kernel must mount the shared ext4 read-only, got: {}",
+            cfg.boot_args
+        );
+        assert!(
+            !cfg.boot_args.contains(" rw "),
+            "ext4_overlay must not request a writable root: `{}`",
+            cfg.boot_args
+        );
+        assert!(
+            cfg.boot_args.contains("init=/forkd-init.sh"),
+            "the guest writable layer lives in /forkd-init.sh: `{}`",
+            cfg.boot_args
+        );
+        // Same rationale as quickstart(); a later token would override it.
+        assert!(cfg.boot_args.contains("clocksource=kvm-clock"));
+        let after = cfg
+            .boot_args
+            .split("clocksource=kvm-clock")
+            .nth(1)
+            .unwrap_or("");
+        assert!(
+            !after.contains("clocksource="),
+            "duplicate clocksource token"
+        );
+    }
+
+    /// The writable layer is capped, and the cap is reachable from the host
+    /// because a job that outgrows it must fail its own write rather than
+    /// OOM the guest.
+    #[test]
+    fn with_rw_size_appends_the_cmdline_hint() {
+        let cfg = BootConfig::ext4_overlay("/tmp/k".into(), "/tmp/r.ext4".into(), "/tmp/w".into())
+            .with_rw_size("8g");
+        assert!(
+            cfg.boot_args.contains("forkd.rw_size=8g"),
+            "init reads the cap from the cmdline: `{}`",
+            cfg.boot_args
+        );
+    }
+
     #[test]
     fn volume_device_name_progression() {
         assert_eq!(volume_device_name(0).unwrap(), "vdb");

rootfs-init/forkd-init.sh

@@ -6,15 +6,63 @@ mount -t proc proc /proc 2>/dev/null
 mount -t sysfs sys /sys 2>/dev/null
 mount -t devtmpfs devtmpfs /dev 2>/dev/null
 
-# Mount /tmp as tmpfs so per-sandbox scratch writes (agent logs,
-# socket paths, hint files used by demos) land in this VM's memory
-# instead of the shared on-disk rootfs.ext4. Without this, multiple
-# sandboxes booted from the same snapshot file would concurrently
-# write to the same inode in the same loop-mounted ext4 and corrupt
-# it ("Structure needs cleaning" on the next access).
-# tmpfs state survives BRANCH because it lives in guest RAM, which
-# is what memory.bin captures.
-mount -t tmpfs -o size=256m tmpfs /tmp 2>/dev/null
+# --- writable layer ---------------------------------------------------
+# The rootfs is booted READ-ONLY (root=/dev/vda ro) and is one ext4 file
+# shared by every sandbox restored from this snapshot, so nothing may
+# write it: two guests writing one filesystem with no coordinator corrupt
+# it ("Structure needs cleaning", foreign bytes inside package files).
+# Everything writable is therefore provided here, in guest RAM.
+#
+# RAM is the point, not a limitation: tmpfs and overlayfs-upper pages are
+# guest memory, and memory.bin is what a snapshot captures, so writable
+# state survives a BRANCH exactly as the /tmp tmpfs always has.
+#
+#   plain tmpfs  /run /dev/shm      -- scratch that owns nothing
+#   overlay      /etc /root /home   -- the image's baked content stays
+#                /opt /srv           visible as the lower layer, writes
+#                /usr/local /var     land in the tmpfs upper
+#
+# The overlay list is where jobs actually write (cargo registry and
+# target, pnpm store, mix _build, the checkout itself) while keeping the
+# image's preinstalled content — a bare tmpfs over those paths would hide
+# it. /etc is on the list because the DNS fix below writes resolv.conf.
+#
+# A path that does not exist in the image is skipped, since a read-only
+# root cannot be mkdir'd into. A write outside this set fails with EROFS:
+# loud, and far better than silently mutating the shared base.
+FORKD_RW_SIZE="$(grep -oE 'forkd\.rw_size=[^ ]+' /proc/cmdline | head -1 | cut -d= -f2-)"
+[ -n "$FORKD_RW_SIZE" ] || FORKD_RW_SIZE=2g
+if mount -t tmpfs -o "size=$FORKD_RW_SIZE" tmpfs /tmp 2>/dev/null; then
+    echo "forkd-init: writable layer $FORKD_RW_SIZE (tmpfs at /tmp)"
+else
+    echo "forkd-init: WARN no writable tmpfs at /tmp; sandboxes will be read-only" >&2
+fi
+# Overlay upper/work dirs live on that tmpfs: overlayfs needs them on the
+# same filesystem, and /tmp is the one path we know is writable.
+mkdir -p /tmp/.rw/upper /tmp/.rw/work 2>/dev/null
+
+_forkd_tmpfs() {  # <path>
+    [ -d "$1" ] || return 0
+    mount -t tmpfs -o "size=$FORKD_RW_SIZE" tmpfs "$1" 2>/dev/null || \
+        echo "forkd-init: WARN $1 not writable (tmpfs mount failed)" >&2
+}
+
+_forkd_overlay() {  # <path>
+    [ -d "$1" ] || return 0
+    install -d "/tmp/.rw/upper$1" "/tmp/.rw/work$1" 2>/dev/null
+    if mount -t overlay overlay \
+        -o "lowerdir=$1,upperdir=/tmp/.rw/upper$1,workdir=/tmp/.rw/work$1" "$1" 2>/dev/null; then
+        echo "forkd-init: $1 writable (overlay upper in RAM)"
+    else
+        echo "forkd-init: WARN $1 left read-only (overlay mount failed)" >&2
+    fi
+}
+
+_forkd_tmpfs /run
+_forkd_tmpfs /dev/shm
+for _p in /etc /root /home /opt /srv /usr/local /var; do
+    _forkd_overlay "$_p"
+done
 
 # Make sure PATH covers both Ubuntu (/usr/bin) and official python (/usr/local/bin)
 # images. Subprocess invocations from the agent inherit this.