fix(layer-stack): decide a definition's layer stack once and never format over data

CLAUDE.md

@@ -45,7 +45,7 @@ Copy the closest existing YAML and fill:
 - `teardown[]` — cleanup CLI invocations.
 - `invariants[]` — currently only `no_orphans` is implemented.
 
-Available `await.kind` values: `replica_count`, `disk_state`, `all_uptodate`, `replica_diskless`, `no_tiebreaker`, `sync_clean`, `resource_absent`, `rd_absent`, `vd_size_kib`, `vd_count`, `pvc_capacity`, `pod_md5_invariant`. See the header comment of `tests/operator-harness/replay-runner.sh` for the contract.
+Available `await.kind` values: `replica_count`, `disk_state`, `all_uptodate`, `replica_diskless`, `no_tiebreaker`, `sync_clean`, `resource_absent`, `rd_absent`, `vd_size_kib`, `vd_count`, `rd_layer_stack`, `rd_annotation`, `pvc_capacity`, `pod_md5_invariant`. See the header comment of `tests/operator-harness/replay-runner.sh` for the contract.
 
 ## Running the harness
 

api/v1alpha1/resourcedefinition_types.go

@@ -25,6 +25,8 @@ import (
 // ResourceDefinitionSpec is the desired state of a LINSTOR
 // ResourceDefinition — the named entity from which Resource (replica)
 // instances are spawned. linstor-csi creates one per PVC.
+//
+// +kubebuilder:validation:XValidation:rule="!has(oldSelf.layerStack) || size(oldSelf.layerStack) == 0 || (has(self.layerStack) && size(self.layerStack) > 0)",message="layerStack is decided once: it may be set or replaced, never emptied"
 type ResourceDefinitionSpec struct {
 	// externalName is the user-facing name surfaced by csi (CSI volume id).
 	// Empty means the same as metadata.name.
@@ -73,8 +75,10 @@ type ResourceDefinitionSpec struct {
 	// Order is top-down: the first layer's device is what the
 	// consumer Pod mounts, the last is the raw block device the
 	// storage provider creates.
-	// Empty = inherits from the parent ResourceGroup; both empty =
-	// `["DRBD","STORAGE"]`.
+	// Empty means not decided yet: the controller fixes it once, from
+	// the parent ResourceGroup's stack for a definition that holds no
+	// data, and the satellite builds nothing until it is set. Once set it
+	// is never emptied. See docs/layer-stack.md.
 	// +optional
 	LayerStack []string `json:"layerStack,omitempty"`
 

cmd/controller/main.go

@@ -241,9 +241,10 @@ func main() {
 		os.Exit(1)
 	}
 	if err := (&controller.ResourceDefinitionReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		Store:  st,
+		Client:   mgr.GetClient(),
+		Scheme:   mgr.GetScheme(),
+		Store:    st,
+		Recorder: mgr.GetEventRecorder("blockstor-controller"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "Failed to create controller", "controller", "resourcedefinition")
 		os.Exit(1)

config/crd/bases/blockstor.cozystack.io_resourcedefinitions.yaml

@@ -343,8 +343,10 @@ spec:
                   Order is top-down: the first layer's device is what the
                   consumer Pod mounts, the last is the raw block device the
                   storage provider creates.
-                  Empty = inherits from the parent ResourceGroup; both empty =
-                  `["DRBD","STORAGE"]`.
+                  Empty means not decided yet: the controller fixes it once, from
+                  the parent ResourceGroup's stack for a definition that holds no
+                  data, and the satellite builds nothing until it is set. Once set it
+                  is never emptied. See docs/layer-stack.md.
                 items:
                   type: string
                 type: array
@@ -472,6 +474,11 @@ spec:
                 - volumeNumber
                 x-kubernetes-list-type: map
             type: object
+            x-kubernetes-validations:
+            - message: 'layerStack is decided once: it may be set or replaced, never
+                emptied'
+              rule: '!has(oldSelf.layerStack) || size(oldSelf.layerStack) == 0 ||
+                (has(self.layerStack) && size(self.layerStack) > 0)'
           status:
             description: status defines the observed state of ResourceDefinition
             properties:

config/rbac/role.yaml

@@ -61,3 +61,10 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch

config/rbac/satellite_role.yaml

@@ -103,6 +103,15 @@ rules:
   verbs:
   - create
   - patch
+# The same events through events.k8s.io, which the manager's recorder
+# writes (the LUKS format refused over data, see resource.go).
+- apiGroups:
+  - events.k8s.io
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
 # corev1.Node — read-only access for Bug 283 .res rendering: the
 # dispatcher's peer-address lookup needs each node's routable
 # InternalIP to overcome piraeus-operator's pod-CIDR overwrite of

docs/layer-stack.md

@@ -27,10 +27,26 @@ where it's the network leg).
 | `["DRBD","LUKS","STORAGE"]` | Encrypted at-rest + replicated. Per-volume cipher.                  |
 | `["STORAGE"]`               | Single-replica local mode. Ephemeral cache, scratch, ZFS dataset.   |
 
-The default is `["DRBD","STORAGE"]` — when `RD.Spec.LayerStack` is
-empty the controller inherits from the parent
-`ResourceGroup.Spec.SelectFilter.LayerStack`, and when both are empty
-the dispatcher falls through to the satellite's default-DRBD path.
+The default is `["DRBD","STORAGE"]`, used when neither the RD nor its resource group sets a stack.
+
+An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The REST create and `rg spawn` paths copy the group's stack onto the RD when it is created. An RD that ends up with an empty stack in a group that sets one (created with `kubectl apply` or GitOps, or moved by `rd modify --resource-group`) is pinned by the controller, and the satellite builds nothing for it until then:
+
+- An RD that holds no data yet gets the group's stack.
+- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. The controller re-checks every RD with a decided stack each time it reconciles it: the annotation names the group's current stack whenever the RD's stack lacks a layer of it, set by hand or pinned, and goes once the group no longer asks for a missing layer or is deleted. A newly flagged RD gets the same warning Event.
+
+Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack, unless that is the default stack and the source carries the annotation, which is how a stack pinned over data looks. Such a clone, and one of a source with no stack, is left for the controller to pin from its own group.
+
+Changing a group's stack later never reshapes RDs that already carry one, as in upstream LINSTOR, where the stack is decided when a resource is created.
+
+As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. The satellite wipes a thick LV it creates before the volume takes its name, so a fresh volume never inherits a signature from extents an earlier volume used; one created outside blockstor may, so clear it with `wipefs -a <device>` once you have checked it holds nothing you need.
+
+Find the RDs short of a layer their group asks for:
+
+```bash
+kubectl get resourcedefinitions.blockstor.cozystack.io \
+  -o custom-columns='NAME:.metadata.name,GROUP_STACK:.metadata.annotations.blockstor\.io/unapplied-group-layer-stack' \
+  | awk '$2 != "<none>"'
+```
 
 ## Setting the stack
 

docs/linstor-migration.md

@@ -10,7 +10,7 @@ Migrated: nodes, storage pools, resource groups, resource definitions (+ volume
 
 **NOT migrated — read the report on stderr for the exact list per cluster:**
 
-- **LUKS passphrases.** LUKS-encrypted volumes convert with their layer stack intact, but the per-volume passphrase (encrypted with the LINSTOR master key) is not decrypted. You must provision `spec.encryption` for those RDs by hand before the encrypted volumes can be opened. Each affected RD is reported.
+- **LUKS passphrases.** LUKS-encrypted volumes convert with their layer stack intact (an RD whose `layer_stack` column is empty takes the stack its replicas were built with, read from `LAYER_RESOURCE_IDS`), but the per-volume passphrase (encrypted with the LINSTOR master key) is not decrypted. You must provision `spec.encryption` for those RDs by hand before the encrypted volumes can be opened. Each affected RD is reported.
 - **Backup-shipping remotes**, unknown/runtime flag bits, and DELETE'd / FAILED_DEPLOYMENT rows are skipped and reported. None affect live data availability.
 - **DRBD TCP ports** are absent from most LINSTOR 1.33 dumps — see "Capture live DRBD ports" below.
 

internal/cli/snapshot.go

@@ -336,7 +336,10 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error {
 		return err
 	}
 
-	def := newRestoredDefinition(args.toResource, &src, &snap)
+	def, err := newRestoredDefinition(ctx, run.Store, args.toResource, &src, &snap)
+	if err != nil {
+		return fmt.Errorf("restore %s onto %s: %w", snap.Name, args.toResource, err)
+	}
 
 	err = run.Store.ResourceDefinitions().Create(ctx, def)
 	if errors.Is(err, store.ErrAlreadyExists) {
@@ -365,18 +368,39 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error {
 }
 
 // newRestoredDefinition is the definition a restore creates: the source's
-// group and stack, the snapshot's props, and this restore's marker.
-func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *apiv1.Snapshot) *apiv1.ResourceDefinition {
+// group and stack, the snapshot's props, and this restore's marker. The stack
+// is the one the source's bytes were laid down under
+// (store.EffectiveLayerStack); a source whose group has yet to decide it is an
+// error the operator retries once the controller pins it. A copy short of what
+// its group asks for is flagged the way the controller flags a definition it
+// pinned over data.
+func newRestoredDefinition(
+	ctx context.Context, st store.Store, rdName string, src *apiv1.ResourceDefinition, snap *apiv1.Snapshot,
+) (*apiv1.ResourceDefinition, error) {
+	stack, err := store.EffectiveLayerStack(ctx, st, src)
+	if err != nil {
+		return nil, fmt.Errorf("the layer stack of the source %s: %w", src.Name, err)
+	}
+
+	unapplied, err := store.UnappliedGroupStack(ctx, st, src.ResourceGroupName, stack)
+	if err != nil {
+		return nil, err //nolint:wrapcheck // names the group itself
+	}
+
 	def := &apiv1.ResourceDefinition{
 		Name: rdName,
 		// The parent group carries over: it drives later placement and
 		// property inheritance, and a restored definition with a blank
 		// group breaks the restore-then-list workflow.
 		ResourceGroupName: src.ResourceGroupName,
-		LayerStack:        store.EffectiveLayerStack(src.LayerStack),
+		LayerStack:        stack,
 		Props:             store.TravellingProps(snap.Props),
 	}
 
+	if unapplied != "" {
+		def.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: unapplied}
+	}
+
 	if def.Props == nil {
 		def.Props = store.TravellingProps(src.Props)
 	}
@@ -387,7 +411,7 @@ func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *a
 	// the placer looks the source half up as a store key.
 	def.Props = store.WithRestoreMarker(def.Props, snap)
 
-	return def
+	return def, nil
 }
 
 // restoreIntoExistingTarget finishes a restore over a definition that is

internal/cli/snapshot_restore_prepared_test.go

@@ -1184,20 +1184,54 @@ func TestSnapshotRestorePromotesATieBreakerOnARequestedNode(t *testing.T) {
 	}
 }
 
-// A source stored without a stack is brought up with the default whatever its
-// group says now; the restored definition carries that stack, stamped.
-func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) {
+func seedUndecidedSnapshotSource(ctx context.Context, backend store.Store) {
+	seedSnapshotSource(ctx, backend)
+	_ = backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{
+		Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}},
+	})
+
+	// Patched rather than updated: Update keeps a decided stack, and the
+	// source stands for one stored without a stack.
+	_ = backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "pvc-x", func(src *apiv1.ResourceDefinition) error {
+		src.LayerStack = nil
+		src.ResourceGroupName = "grp-luks"
+
+		return nil
+	})
+}
+
+// A source stored without a stack under a group that sets one has its stack
+// decided by the controller, which has not done it yet: the restore refuses,
+// saying so, and creates nothing.
+func TestSnapshotRestoreRefusesAnUndecidedSourceStack(t *testing.T) {
+	t.Parallel()
+
+	app, _, errBuf := newApp(t, seedUndecidedSnapshotSource)
+
+	if got := app.Run(t.Context(), restoreResourceArgv("pvc-stack")); got == 0 {
+		t.Fatal("restore of an undecided source exited 0")
+	}
+
+	if !strings.Contains(errBuf.String(), "not pinned yet") {
+		t.Errorf("stderr = %q, want it to say the stack is not pinned yet", errBuf.String())
+	}
+
+	if _, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-stack"); err == nil {
+		t.Error("the refused restore created the target")
+	}
+}
+
+// Once the controller pins the source over its data to the default stack, the
+// restored definition carries that stack and is flagged as short of its
+// group's.
+func TestSnapshotRestoreStampsTheSourcesPinnedStack(t *testing.T) {
 	t.Parallel()
 
 	app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) {
-		seedSnapshotSource(ctx, backend)
-		_ = backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{
-			Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}},
-		})
+		seedUndecidedSnapshotSource(ctx, backend)
 
 		src, _ := backend.ResourceDefinitions().Get(ctx, "pvc-x")
-		src.LayerStack = nil
-		src.ResourceGroupName = "grp-luks"
+		src.LayerStack = apiv1.DefaultLayerStack()
 		_ = backend.ResourceDefinitions().Update(ctx, &src)
 	})
 
@@ -1211,7 +1245,11 @@ func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) {
 	}
 
 	if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) {
-		t.Errorf("restored stack = %v, want the source's data-plane %v", rd.LayerStack, apiv1.DefaultLayerStack())
+		t.Errorf("restored stack = %v, want the source's %v", rd.LayerStack, apiv1.DefaultLayerStack())
+	}
+
+	if got := rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; got != "DRBD,LUKS,STORAGE" {
+		t.Errorf("unapplied-group annotation = %q, want DRBD,LUKS,STORAGE", got)
 	}
 }
 

internal/controller/export_test.go

@@ -68,14 +68,6 @@ func AlreadyExists(err error) bool {
 	return alreadyExists(err)
 }
 
-// ResolveLayerStack exposes the four-tier layer-stack resolver:
-// RD spec → RG spec → nil (dispatcher default). Tests pin each
-// tier so a refactor that swapped precedence (RG over RD, say)
-// would surface immediately.
-func (r *ResourceReconciler) ResolveLayerStack(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) []string {
-	return r.resolveLayerStack(ctx, rd)
-}
-
 // ResolveEffectiveProps exposes the four-tier prop resolver: cluster
 // ControllerProps → RG → RD → Resource. Tests pin each tier and the
 // soft-fail-on-missing-RG path.
@@ -248,3 +240,6 @@ const AutoTiebreakerSuppressedUntilAnnotation = apiv1.AutoTiebreakerSuppressedUn
 // annotation key for tests. Same compile-time-rename guard as the
 // suppression key above.
 const KeepTiebreakerUntilAnnotation = apiv1.KeepTiebreakerUntilAnnotation
+
+// RDReconcileRequeue exposes the RD reconciler's periodic requeue.
+const RDReconcileRequeue = rdReconcileRequeue

internal/controller/group_layer_stack_watch_test.go

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/*
+Copyright 2026 Cozystack contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+
+	blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1"
+)
+
+func watchGroup(stack ...string) *blockstoriov1alpha1.ResourceGroup {
+	return &blockstoriov1alpha1.ResourceGroup{
+		ObjectMeta: metav1.ObjectMeta{Name: "rg-watch"},
+		Spec: blockstoriov1alpha1.ResourceGroupSpec{
+			SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{LayerStack: stack},
+		},
+	}
+}
+
+// Only a change to a group's layer stack re-checks its RDs.
+func TestGroupLayerStackChangedFiresOnlyOnAStackChange(t *testing.T) {
+	t.Parallel()
+
+	pred := groupLayerStackChanged()
+
+	if !pred.Update(event.UpdateEvent{ObjectOld: watchGroup("DRBD", "STORAGE"), ObjectNew: watchGroup("DRBD", "LUKS", "STORAGE")}) {
+		t.Error("a group that gained LUKS did not re-check its RDs")
+	}
+
+	if pred.Update(event.UpdateEvent{ObjectOld: watchGroup("DRBD", "STORAGE"), ObjectNew: watchGroup("DRBD", "STORAGE")}) {
+		t.Error("a group update that left the stack alone re-checked its RDs")
+	}
+}
+
+// The group's RDs are enqueued under any spelling of its name, and no other.
+func TestEnqueueRDsInGroupFindsItsRDsOnly(t *testing.T) {
+	t.Parallel()
+
+	rd := func(name, group string) *blockstoriov1alpha1.ResourceDefinition {
+		return &blockstoriov1alpha1.ResourceDefinition{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Spec:       blockstoriov1alpha1.ResourceDefinitionSpec{ResourceGroupName: group},
+		}
+	}
+
+	cli := fake.NewClientBuilder().WithScheme(bug048Scheme(t)).
+		WithObjects(rd("pvc-a", "rg-watch"), rd("pvc-b", "RG-Watch"), rd("pvc-c", "rg-other"), rd("pvc-d", "")).Build()
+	r := &ResourceDefinitionReconciler{Client: cli}
+
+	got := map[string]bool{}
+	for _, req := range r.enqueueRDsInGroup(context.Background(), watchGroup()) {
+		got[req.Name] = true
+	}
+
+	if len(got) != 2 || !got["pvc-a"] || !got["pvc-b"] {
+		t.Errorf("enqueued %v, want pvc-a and pvc-b", got)
+	}
+}