test(nvsnap): refuse to measure a cold start as a restore

src/compute-plane-services/nvsnap/cmd/agent/main.go

@@ -116,6 +116,8 @@ func main() {
 		"Directory the Local backend writes captures into")
 	flag.StringVar(&config.RootfsCapture.PodCacheDir, "pod-cache-dir", os.Getenv("NVSNAP_POD_CACHE_DIR"),
 		"In-pod cache mount path (e.g. /opt/nvsnap) for cachedir mode: capture ONLY this dir as the PVC root, restore RO-mounts the rox here (no overlayfs). Empty = standard whole-rootfs capture. Must match the webhook's cacheDir.")
+	flag.BoolVar(&config.RootfsCapture.AllowWholeRootfs, "allow-whole-rootfs", os.Getenv("NVSNAP_ALLOW_WHOLE_ROOTFS") == "1",
+		"Permit capture without --pod-cache-dir, i.e. capture the whole container rootfs. Off by default: whole-rootfs capture succeeds silently and only diverges later, at restore, from the cachedir behaviour every workload and benchmark assumes. Set only to run that path deliberately.")
 	flag.StringVar(&config.RootfsCapture.PodCacheEnvFile, "cachedir-env-file", os.Getenv("NVSNAP_CACHEDIR_ENV_FILE"),
 		"Path to a mounted ConfigMap file with the cachedir env template (NAME=value lines; {root}/{cache}/{model} placeholders). Read on capture inject only — edit the ConfigMap to add/remove cache env vars without an agent rebuild. Empty/unreadable = built-in default. Restore replays the env stamped in the manifest.")
 	flag.StringVar(&config.OverlayRoot, "overlay-root", "/var/lib/nvsnap/overlays",

src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gpt-oss-120b-restore.yaml

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# Restore pod for PDF-bench gpt-oss-120b (TP=4, rootfs path).
+# Restore pod for PDF-bench gpt-oss-120b (TP=4, cachedir path).
 #
 # Customer-shape: no CRIU init ladder, no nodeName pin. The nvsnap
 # webhook injects PVC mounts and (for Local backend) nodeAffinity

src/compute-plane-services/nvsnap/deploy/k8s/benchmarks/gpt-oss-120b.yaml

@@ -20,7 +20,7 @@
 #                 --no-enable-prefix-caching
 #   Hardware    : 4x H100 (one node, TP=4)
 #
-# Multi-GPU → rootfs capture path per CLAUDE.md rule 20. No CRIU init
+# Multi-GPU → cachedir capture path per CLAUDE.md rule 20. No CRIU init
 # container ladder; just the workload + nvsnap.io/capture=true label.
 # The agent's rootfsonly.Watcher snapshots the overlay upperdir after
 # the readiness probe passes + 60s warmup.
@@ -33,10 +33,10 @@ metadata:
     app: bench-gpt-oss-120b
     nvsnap.io/bench: "pdf-matrix"
     nvsnap.io/bench-row: "llm-medium"
-    nvsnap.io/capture: "true"   # opt-in to rootfs capture (multi-GPU only)
+    nvsnap.io/capture: "true"   # opt-in to cachedir capture (multi-GPU only)
   annotations:
     nvsnap.io/desc: "PDF bench: openai/gpt-oss-120b TP=4 on vllm:v0.20.0"
-    nvsnap.io/path: "rootfs"
+    nvsnap.io/path: "cachedir"
 spec:
   tolerations:
     - key: "nvidia.com/gpu"

src/compute-plane-services/nvsnap/deploy/k8s/workloads/nim-qwen3-32b.yaml

@@ -15,14 +15,14 @@ metadata:
   labels:
     app: nim-qwen3-32b
     nvsnap.io/demo: "true"
-    nvsnap.io/capture: "true"   # opt-in to rootfs capture (multi-GPU only)
+    nvsnap.io/capture: "true"   # opt-in to cachedir capture (multi-GPU only)
   annotations:
     nvsnap.io/demo-name: "NIM"
-    nvsnap.io/desc: "Qwen3-32B multi-GPU TP=2 (rootfs)"
+    nvsnap.io/desc: "Qwen3-32B multi-GPU TP=2 (cachedir)"
     nvsnap.io/model: "qwen/qwen3-32b"
     nvsnap.io/port: "8000"
     nvsnap.io/gpus: "2"
-    nvsnap.io/path: "rootfs"
+    nvsnap.io/path: "cachedir"
     nvsnap.io/ckpt-size: "61 GB"
 spec:
   tolerations:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-70b.yaml

@@ -6,9 +6,9 @@
 # nvsnap agent's rootfs watcher captures this pod after the warmup window.
 #
 # Customer-shape minimal yaml — no CRIU init container ladder. The
-# rootfs path captures the pod's overlay upperdir + Hugging Face cache
-# + compiled engine caches and replays them via webhook-injected bind
-# mounts on the corresponding `vllm-70b-fresh` pod.
+# cachedir path captures ONLY the pod's cache mount (Hugging Face model
+# + compiled engine caches), not the whole container rootfs, and replays
+# it via webhook-injected mounts on the corresponding restore pod.
 apiVersion: v1
 kind: Pod
 metadata:
@@ -17,14 +17,14 @@ metadata:
   labels:
     app: vllm-70b
     nvsnap.io/demo: "true"
-    nvsnap.io/capture: "true"   # opt-in to rootfs capture (multi-GPU only)
+    nvsnap.io/capture: "true"   # opt-in to cachedir capture (multi-GPU only)
   annotations:
     nvsnap.io/demo-name: "vLLM"
-    nvsnap.io/desc: "Llama-3.1-70B multi-GPU tensor-parallel (TP=4, rootfs)"
+    nvsnap.io/desc: "Llama-3.1-70B multi-GPU tensor-parallel (TP=4, cachedir)"
     nvsnap.io/model: "meta-llama/Llama-3.1-70B-Instruct"
     nvsnap.io/port: "8000"
     nvsnap.io/gpus: "4"
-    nvsnap.io/path: "rootfs"
+    nvsnap.io/path: "cachedir"
     nvsnap.io/ckpt-size: "132 GB"
 spec:
   tolerations:

src/compute-plane-services/nvsnap/deploy/k8s/workloads/vllm-tp2.yaml

@@ -33,7 +33,7 @@ metadata:
     # anything requesting >= 2 GPUs here. Declaring "criu" made the manifest
     # generator emit a criu-v2 restore placeholder that nothing ever drives,
     # so the restore pod idled until the readiness timeout.
-    nvsnap.io/path: "rootfs"
+    nvsnap.io/path: "cachedir"
 spec:
   automountServiceAccountToken: false
   tolerations:

src/compute-plane-services/nvsnap/internal/agent/BUILD.bazel

@@ -107,6 +107,7 @@ go_test(
         "restoreoverlay_integration_test.go",
         "restoreoverlay_test.go",
         "rootfs_diff_test.go",
+        "rootfs_wholerootfs_guard_test.go",
         "rootfsonly_integration_test.go",
     ],
     embed = [":agent"],

src/compute-plane-services/nvsnap/internal/agent/rootfs_wholerootfs_guard_test.go

@@ -0,0 +1,67 @@
+/*
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package agent
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// Whole-rootfs capture must not start by accident. It succeeds quietly --
+// producing a capture that restores -- so a cluster whose cachedir setting was
+// dropped keeps running while diverging from every workload and benchmark that
+// assumes cachedir. The agent refuses at startup instead.
+func TestStartRootfsCaptureRefusesWholeRootfs(t *testing.T) {
+	a := &Agent{}
+	_, err := a.startRootfsCapture(context.Background(), RootfsCaptureConfig{
+		Enabled: true, // no PodCacheDir, no override
+	})
+	if err == nil {
+		t.Fatal("expected refusal when --pod-cache-dir is unset; whole-rootfs must be opt-in")
+	}
+	for _, want := range []string{"pod-cache-dir", "allow-whole-rootfs"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("error should tell the operator about %q, got: %v", want, err)
+		}
+	}
+}
+
+// The override exists so an operator can still run that path deliberately.
+// It must get past the guard -- failing later for an unrelated reason (no
+// kube client in a unit test) is fine; failing *at the guard* is not.
+func TestStartRootfsCaptureAllowsExplicitOptIn(t *testing.T) {
+	a := &Agent{}
+	_, err := a.startRootfsCapture(context.Background(), RootfsCaptureConfig{
+		Enabled:          true,
+		AllowWholeRootfs: true,
+	})
+	if err != nil && strings.Contains(err.Error(), "whole-rootfs capture is not supported") {
+		t.Fatalf("explicit opt-in must pass the guard, got: %v", err)
+	}
+}
+
+// Disabled stays a clean no-op: the guard must not turn "capture off" into an
+// error for every agent that does not run capture at all.
+func TestStartRootfsCaptureDisabledIsNoop(t *testing.T) {
+	a := &Agent{}
+	b, err := a.startRootfsCapture(context.Background(), RootfsCaptureConfig{Enabled: false})
+	if err != nil || b != nil {
+		t.Fatalf("disabled capture should be a no-op, got backend=%v err=%v", b, err)
+	}
+}

src/compute-plane-services/nvsnap/internal/agent/rootfsonly_integration.go

@@ -57,6 +57,16 @@ type RootfsCaptureConfig struct {
 	// replays the env stamped into the manifest.
 	PodCacheEnvFile string
 
+	// AllowWholeRootfs permits capture to run with PodCacheDir empty, i.e.
+	// capturing the whole container rootfs instead of just the cache mount.
+	//
+	// Off by default, and the agent refuses to start capture without it,
+	// because the failure mode is silent: whole-rootfs looks like a working
+	// capture, and the difference only surfaces later as restores that
+	// behave unlike the ones that were benchmarked. Requiring an explicit
+	// opt-in makes running it a decision rather than an oversight.
+	AllowWholeRootfs bool
+
 	// CMNamespace is the K8s namespace ConfigMaps are written to so
 	// any node's webhook can resolve a hash. Default "nvsnap-system".
 	CMNamespace string
@@ -84,6 +94,19 @@ func (a *Agent) startRootfsCapture(ctx context.Context, cfg RootfsCaptureConfig)
 	if !cfg.Enabled {
 		return nil, nil
 	}
+	// Refuse whole-rootfs capture unless explicitly allowed. Capturing the
+	// entire container rootfs still "works" -- it produces a capture, restores
+	// succeed, and nothing looks wrong -- so a cluster that lost its cachedir
+	// setting would keep running and silently diverge from every workload and
+	// benchmark that assumes cachedir. Fail at startup, where an operator sees
+	// it, rather than at restore time, where it looks like a performance
+	// mystery.
+	if cfg.PodCacheDir == "" && !cfg.AllowWholeRootfs {
+		return nil, fmt.Errorf("rootfs capture is enabled without --pod-cache-dir: " +
+			"whole-rootfs capture is not supported for normal use. Set --pod-cache-dir " +
+			"(e.g. /opt/nvsnap) to capture the cache mount, or pass --allow-whole-rootfs " +
+			"to override deliberately")
+	}
 	if cfg.CacheDir == "" {
 		cfg.CacheDir = "/var/lib/nvsnap/cache"
 	}

src/compute-plane-services/nvsnap/internal/agent/rootfsonly_integration_test.go

@@ -19,6 +19,7 @@ package agent
 
 import (
 	"context"
+	"strings"
 	"testing"
 
 	"github.com/sirupsen/logrus"
@@ -38,8 +39,17 @@ func TestStartRootfsCapture_EnabledFailsWithoutKubeConfig(t *testing.T) {
 	t.Setenv("KUBECONFIG", "/nonexistent/kubeconfig")
 	t.Setenv("HOME", t.TempDir()) // hide any ~/.kube/config the test runner has
 	a := &Agent{config: Config{}, log: logrus.New()}
-	_, err := a.startRootfsCapture(context.Background(), RootfsCaptureConfig{Enabled: true})
+	// PodCacheDir is set so the whole-rootfs guard does not answer first.
+	// Without it this test would still fail -- but on the guard, not on the
+	// kube client it is named for, and the coverage would be gone silently.
+	_, err := a.startRootfsCapture(context.Background(), RootfsCaptureConfig{
+		Enabled:     true,
+		PodCacheDir: "/opt/nvsnap",
+	})
 	if err == nil {
 		t.Fatal("expected kube client construction error when no config available")
 	}
+	if strings.Contains(err.Error(), "whole-rootfs") {
+		t.Fatalf("guard fired instead of the kube client path; this test no longer covers what it claims: %v", err)
+	}
 }

src/compute-plane-services/nvsnap/internal/server/manifests.go

@@ -84,7 +84,20 @@ type CapturePath string
 
 // Capture path identifiers.
 const (
-	CapturePathCRIU   CapturePath = "criu"
+	CapturePathCRIU CapturePath = "criu"
+
+	// CapturePathCacheDir captures only the pod's cache mount (model +
+	// compile caches) rather than the whole container rootfs. This is what
+	// multi-GPU workloads use.
+	CapturePathCacheDir CapturePath = "cachedir"
+
+	// CapturePathRootfs captures the whole container rootfs.
+	//
+	// Deprecated: not used by any workload. It was the original multi-GPU
+	// path, and manifests kept declaring "rootfs" long after the agent's
+	// cachedir mode meant they were really capturing only the cache dir --
+	// so the label described a path that was not running. That mismatch
+	// cost real debugging time. Kept only so an older manifest still parses.
 	CapturePathRootfs CapturePath = "rootfs"
 )