feat(vmm): give every restored child its own rootfs backing

CHANGELOG.md

@@ -14,6 +14,60 @@ Versioning](https://semver.org/spec/v2.0.0.html) once it reaches
 chain-assembly base copy and the bake's rootfs baseline clone streamed a full
 copy even on btrfs, XFS, and ZFS 2.2+. Found by @jrimmer in #321.
 
+### Per-child rootfs backings: concurrent children can no longer corrupt each other
+
+A snapshot's rootfs is one ext4 and Firecracker reopens that path verbatim for
+every child, so children of a tag shared a filesystem. With the drive opened
+read-write — how every tag was baked — two concurrent children were two guest
+kernels writing one filesystem with no coordinator: package files picking up
+other files' bytes, `EBADMSG` on `/var/lib/dpkg` entries, damage that reads as a
+random build failure rather than a sandbox error.
+
+Each child now gets its own reflink clone of the tag's rootfs and is re-pointed
+at it with `PATCH /drives/rootfs` before it runs. The ordering is load-bearing:
+the child is loaded `resume_vm: false`, re-pointed, and only then resumed, so no
+boot-time write (journal replay, `/var/log`, agent startup) can reach the shared
+base first. Measured on v1.17.0, the base stays byte-identical through boot
+while the backing changes.
+
+**Fail closed.** After the PATCH, forkd reads `GET /vm/config` back and resumes
+the child only if Firecracker reports the child's own backing as its rootfs. A
+child whose rootfs cannot be re-pointed and verified is not resumed — it would
+run against the shared base — so the failure surfaces instead of drifting into
+corruption. Backings live in the child's work dir and are reclaimed with it.
+
+**No Firecracker upgrade is required.** Re-pointing a drive on a restored VM
+works on every stock release measured — v1.5.1, v1.7.0, v1.10.1, v1.12.1,
+v1.14.1 and v1.17.0 (aarch64): the guest's writes move to the new file and
+`/vm/config` reports it. So existing snapshots keep loading and nothing needs
+re-baking.
+
+**Read-only rootfs is shared, not copied.** Whether the drive is writable is
+read from the restored vmstate itself (`/vm/config`), so a read-only (squashfs)
+tag — including one baked before this change — costs no copy per child.
+
+**Branches and workspace suspends own their rootfs.** A snapshot of a restored
+child used to record the child's backing, a file in a work dir that dies with
+the child: the new tag failed to load (`Failed to restore from snapshot … No
+such file`) once its source was gone, and until then every child restored from
+it wrote the source's live disk. The paused source's drive is now pointed at a
+clone the new tag owns for the length of the snapshot, recorded as the tag's
+`rootfs`, and pointed home before the source resumes; a source whose drive
+cannot be pointed home is killed rather than resumed onto the tag's disk. This
+covers full, diff and live BRANCH and `POST /v1/workspaces/:name/suspend`.
+
+Constraints worth knowing: a reflink clone is free only where the filesystem
+supports it (XFS/btrfs, or ZFS 2.2+ block cloning); elsewhere it is a full copy
+per child, made in parallel in the children's restore threads. A live BRANCH
+makes its clone inside its pause window, so on a filesystem without reflink the
+pause includes a full rootfs copy. Snapshots with no recorded rootfs keep the
+previous single-call behaviour.
+
+Restore work dirs, and so the backings in them, live under the controller's
+temp dir. On a host where `/tmp` is tmpfs, set `TMPDIR` to a directory on the
+snapshot root's filesystem, so backings are not charged to RAM and the clone
+has both ends on one filesystem.
+
 ### Upgrade note: legacy sandbox rows block startup
 
 The controller now persists a boot identity (start time + boot id) for every

crates/forkd-cli/src/main.rs

@@ -4830,6 +4830,14 @@ mod tests {
             .unwrap();
     }
 
+    /// Pids the sweep tests park backups under, standing for bakes that have
+    /// exited. They must never name a live process, or the sweep leaves the
+    /// backup alone as a concurrent bake's: small made-up pids like 1000 are
+    /// often live on a CI runner. The kernel never hands out a pid above
+    /// `PID_MAX_LIMIT` (2^22 on 64-bit), so these cannot be.
+    const DEAD_PID_A: u32 = 4_194_305;
+    const DEAD_PID_B: u32 = 4_194_306;
+
     /// Review #295 approval follow-up (2026-09-02): a `kill -9` between
     /// the preserve-rename and the publish strands the backup under the
     /// dead run's pid, leaving the tag with no `rootfs.ext4`. The sweep
@@ -4839,8 +4847,12 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let snap_dir = tmp.path().join("tag");
         std::fs::create_dir_all(&snap_dir).unwrap();
-        let older = tmp.path().join("tag.rootfs.ext4.prev-1000");
-        let newer = tmp.path().join("tag.rootfs.ext4.prev-2000");
+        let older = tmp
+            .path()
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_A}"));
+        let newer = tmp
+            .path()
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_B}"));
         write_with_mtime(&older, b"OLD-ROOTFS", 1_700_000_000);
         // Ordering is by park time (ctime), so park the newer one later.
         std::thread::sleep(std::time::Duration::from_millis(20));
@@ -4871,8 +4883,12 @@ mod tests {
         std::fs::create_dir_all(&snap_dir).unwrap();
         let live = snap_dir.join("rootfs.ext4");
         std::fs::write(&live, b"CURRENT-GOOD").unwrap();
-        let a = tmp.path().join("tag.rootfs.ext4.prev-1000");
-        let b = tmp.path().join("tag.rootfs.ext4.prev-2000");
+        let a = tmp
+            .path()
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_A}"));
+        let b = tmp
+            .path()
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_B}"));
         write_with_mtime(&a, b"STALE-A", 1_700_000_000);
         write_with_mtime(&b, b"STALE-B", 1_700_000_900);
         let now = std::time::SystemTime::now()
@@ -4905,7 +4921,9 @@ mod tests {
         let snap_dir = tmp.path().join("tag");
         std::fs::create_dir_all(&snap_dir).unwrap();
         write_with_mtime(&snap_dir.join("snapshot.json"), b"{}", 1_700_000_000);
-        let backup = tmp.path().join("tag.rootfs.ext4.prev-1000");
+        let backup = tmp
+            .path()
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_A}"));
         write_with_mtime(&backup, b"LAST-PUBLISHED", 1_700_000_000);
         std::fs::write(snap_dir.join("rootfs.ext4"), b"UNPUBLISHED-DIRTY-CLONE").unwrap();
 
@@ -4959,11 +4977,11 @@ mod tests {
         let held = tmp
             .path()
             .join("snapshots")
-            .join("tag.rootfs.ext4.prev-1000");
+            .join(format!("tag.rootfs.ext4.prev-{DEAD_PID_A}"));
         let other = tmp
             .path()
             .join("snapshots")
-            .join("other.rootfs.ext4.prev-1000");
+            .join(format!("other.rootfs.ext4.prev-{DEAD_PID_A}"));
         std::fs::write(&held, b"MINE").unwrap();
         std::fs::write(&other, b"OTHER-TAG").unwrap();
 

crates/forkd-controller/src/http.rs

@@ -482,11 +482,22 @@ async fn create_snapshot(
         // Volumes via daemon snapshot API will land in a follow-up commit;
         // for now snapshots created through the daemon are volume-less.
         // Use the CLI's `forkd snapshot --volume` for tag-shared caches.
-        let snap = vm.snapshot_to(
+        let mut snap = vm.snapshot_to(
             snap_dir_for_task.join("vmstate"),
             snap_dir_for_task.join("memory.bin"),
             Vec::new(),
         )?;
+        // Record the rootfs this parent booted from. Restore gives every child
+        // its own reflink backing cloned from this path, and with no path there
+        // is nothing to clone, so children fall back to sharing one file — the
+        // daemon's snapshots were the last ones doing that. Branches and
+        // workspace suspends record a clone of their own (see
+        // `Vm::detour_rootfs_into`).
+        snap.rootfs = Some(
+            cfg.rootfs
+                .canonicalize()
+                .unwrap_or_else(|_| cfg.rootfs.clone()),
+        );
         // Persist Snapshot metadata so subsequent forks read back the same
         // (possibly volume-bearing) snapshot description.
         let meta = serde_json::to_vec_pretty(&snap)?;
@@ -1359,7 +1370,10 @@ async fn create_sandbox(
         // ran concurrently (e.g. two branches of the same source). Mix the
         // netns offset in so concurrent batches get distinct work_dirs.
         let work_dir =
-            std::env::temp_dir().join(format!("forkd-daemon-{tag_for_work_dir}-o{netns_offset}"));
+            crate::state::work_dir_root().join(format!(
+                "{}{tag_for_work_dir}-o{netns_offset}",
+                crate::state::WORK_DIR_PREFIX
+            ));
         // v0.5 chain assembly. If `chain` is set, hash-verify each
         // parent against the recorded parent_content_hash (this is
         // the foot-gun guard committed to in the design doc), then
@@ -1975,7 +1989,7 @@ async fn branch_sandbox(
                 // our mmap.
                 #[cfg(target_os = "linux")]
                 if live_mode {
-                    let (live_pause_ms, worker) = run_live_branch_setup(
+                    let (live_pause_ms, worker, branch_rootfs) = run_live_branch_setup(
                         &vm,
                         &snap_dir_for_task,
                         &dst_mem,
@@ -2007,7 +2021,7 @@ async fn branch_sandbox(
                         volumes: source_volumes,
                         parent_tag: None,
                         parent_content_hash: None,
-                        rootfs: None,
+                        rootfs: branch_rootfs,
                     });
                 }
                 #[cfg(not(target_os = "linux"))]
@@ -2038,6 +2052,15 @@ async fn branch_sandbox(
                 let pause_start = std::time::Instant::now();
                 vm.pause()?;
 
+                // A restored child's rootfs is its own backing, which dies
+                // with it. Point the drive at a copy this tag owns for the
+                // snapshot, so the vmstate records a durable path, and back
+                // before the source resumes (see `Vm::detour_rootfs_into`).
+                let rootfs_detour = vm
+                    .detour_rootfs_into(&snap_dir_for_task)
+                    .context("give the branch its own rootfs")?;
+                let branch_rootfs = rootfs_detour.as_ref().map(|d| d.path().to_path_buf());
+
                 // Phase 1a measurement hook: take a Diff snapshot first
                 // (captures pages dirtied since restore; clears the dirty
                 // bitmap). Discarded after metrics. The subsequent Full
@@ -2093,6 +2116,9 @@ async fn branch_sandbox(
                         diff_snap.physical_size_bytes,
                         diff_snap.logical_size_bytes,
                     ));
+                    if let Some(detour) = rootfs_detour {
+                        detour.finish()?;
+                    }
                     let resume_result = vm.resume();
                     pause_ms = Some(pause_start.elapsed().as_millis() as u64);
 
@@ -2138,16 +2164,20 @@ async fn branch_sandbox(
                         volumes: diff_snap.volumes,
                         parent_tag: None,
                         parent_content_hash: None,
-                        rootfs: None,
+                        rootfs: branch_rootfs,
                     }
                 } else {
-                    let snap = vm.snapshot_to(
+                    let mut snap = vm.snapshot_to(
                         snap_dir_for_task.join("vmstate"),
                         snap_dir_for_task.join("memory.bin"),
                         // Inherit volumes from the source snapshot so grandchildren
                         // re-attach the same persistent disks the source had.
                         source_volumes,
                     )?;
+                    if let Some(detour) = rootfs_detour {
+                        detour.finish()?;
+                    }
+                    snap.rootfs = branch_rootfs;
                     // resume() may fail after a successful snapshot. The snapshot file
                     // is intact and usable; the source sandbox is in an unknown state
                     // (most likely still paused). We log and continue rather than
@@ -2607,7 +2637,7 @@ fn run_live_branch_setup(
     dst_mem: &std::path::Path,
     source_volumes: Vec<forkd_vmm::VolumeSpec>,
     id_for_log: &str,
-) -> anyhow::Result<(u64, LiveBranchWorker)> {
+) -> anyhow::Result<(u64, LiveBranchWorker, Option<std::path::PathBuf>)> {
     use std::os::fd::AsRawFd;
 
     let memfd = vm.memfd_handle().ok_or_else(|| {
@@ -2699,12 +2729,23 @@ fn run_live_branch_setup(
     // resume. PauseGuard's Drop resumes on early return.
     let pause_start = std::time::Instant::now();
     let pause_guard = vm.pause_guard()?;
+    // Declared after the pause guard so an early return drops it first: the
+    // drive goes home before the guard resumes the source. The clone runs
+    // inside the pause window — near-free with reflink, a full rootfs copy
+    // without it — because the copy has to match the vmstate's instant.
+    let rootfs_detour = vm
+        .detour_rootfs_into(snap_dir)
+        .context("give the branch its own rootfs")?;
+    let branch_rootfs = rootfs_detour.as_ref().map(|d| d.path().to_path_buf());
     vm.snapshot_vmstate_only(
         snap_dir.join("vmstate"),
         dst_mem.to_path_buf(),
         source_volumes,
     )
     .context("vmstate-only snapshot during live BRANCH")?;
+    if let Some(detour) = rootfs_detour {
+        detour.finish()?;
+    }
     pause_guard.resume().context("resume after vmstate dump")?;
     let pause_ms = pause_start.elapsed().as_millis() as u64;
 
@@ -2720,6 +2761,7 @@ fn run_live_branch_setup(
             mmap_guard,
             wp_branch,
         },
+        branch_rootfs,
     ))
 }
 
@@ -3216,6 +3258,12 @@ async fn suspend_workspace(
                 None
             };
             vm.pause()?;
+            // As in `branch_sandbox`: the state snapshot must own its rootfs,
+            // not record the live child's backing, which dies with the child.
+            let rootfs_detour = vm
+                .detour_rootfs_into(&snap_dir_for_task)
+                .context("give the workspace state its own rootfs")?;
+            let state_rootfs = rootfs_detour.as_ref().map(|d| d.path().to_path_buf());
             let snap = if diff_mode {
                 let diff_path = std::env::temp_dir().join(format!(
                     "forkd-ws-diff-{}-{}.bin",
@@ -3227,6 +3275,9 @@ async fn suspend_workspace(
                     diff_path.clone(),
                     Vec::new(),
                 )?;
+                if let Some(detour) = rootfs_detour {
+                    detour.finish()?;
+                }
                 vm.resume()?;
                 pause_ms = Some(pause_start.elapsed().as_millis() as u64);
                 if let Some(h) = cp_handle {
@@ -3241,14 +3292,18 @@ async fn suspend_workspace(
                     volumes: diff_snap.volumes,
                     parent_tag: None,
                     parent_content_hash: None,
-                    rootfs: None,
+                    rootfs: state_rootfs,
                 }
             } else {
-                let snap = vm.snapshot_to(
+                let mut snap = vm.snapshot_to(
                     snap_dir_for_task.join("vmstate"),
                     snap_dir_for_task.join("memory.bin"),
                     Vec::new(),
                 )?;
+                if let Some(detour) = rootfs_detour {
+                    detour.finish()?;
+                }
+                snap.rootfs = state_rootfs;
                 vm.resume()?;
                 pause_ms = Some(pause_start.elapsed().as_millis() as u64);
                 snap