From 4fc1c98ba6f5a2bd26a270727dd703487b17ade4 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 12 Oct 2026 23:55:24 +0200 Subject: [PATCH 01/22] fix(controller): pin a definition's layer stack from its group once A definition that leaves its layer stack empty and sits in a group that sets one was read two ways: the REST and witness gates resolved the group's stack, while the satellite built the volume from the empty one, which is the default stack. In a LUKS group that put plaintext on disk under a definition every gate called encrypted. The stack is now decided once, by the controller, and written onto the definition, as upstream decides it at resource creation. A definition with nothing on disk takes the group's stack. One that already holds data keeps the stack its storage was built with: giving it the group's now would have the satellite build a different stack over that data. It is annotated with the group's stack it did not get, and a warning event names it, so such a volume reads as what it is and can be found. The group is looked up under its folded name, since a definition keeps the group's name as the operator spelled it. Once set, the stack is never emptied again. Both stores keep a decided stack when an update carries none, as a caller writing back a copy read before the pin does, and the CRD refuses an update that empties it, so the controller never decides a second time over a volume that may by then hold data. A stack can still be replaced by another one. The spec.layerStack description says what an empty stack now means. The controller-side resolver that walked the group on read had no callers left and is removed. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- api/v1alpha1/resourcedefinition_types.go | 8 +- cmd/controller/main.go | 7 +- ...stor.cozystack.io_resourcedefinitions.yaml | 11 +- config/rbac/role.yaml | 7 + internal/controller/export_test.go | 11 +- internal/controller/layer_stack_test.go | 155 -------- internal/controller/resource_controller.go | 26 -- .../resourcedefinition_controller.go | 21 +- .../resourcedefinition_layer_stack.go | 154 +++++++ .../resourcedefinition_layer_stack_test.go | 376 ++++++++++++++++++ pkg/api/v1/annotations.go | 9 + pkg/store/inmemory_resource_definition.go | 5 + pkg/store/k8s/layer_stack_test.go | 68 ++++ pkg/store/k8s/resource_definitions.go | 10 + pkg/store/storetest/storetest.go | 49 +++ 15 files changed, 717 insertions(+), 200 deletions(-) delete mode 100644 internal/controller/layer_stack_test.go create mode 100644 internal/controller/resourcedefinition_layer_stack.go create mode 100644 internal/controller/resourcedefinition_layer_stack_test.go create mode 100644 pkg/store/k8s/layer_stack_test.go diff --git a/api/v1alpha1/resourcedefinition_types.go b/api/v1alpha1/resourcedefinition_types.go index 833134c9..ea62f436 100644 --- a/api/v1alpha1/resourcedefinition_types.go +++ b/api/v1alpha1/resourcedefinition_types.go @@ -25,6 +25,8 @@ import ( // ResourceDefinitionSpec is the desired state of a LINSTOR // ResourceDefinition — the named entity from which Resource (replica) // instances are spawned. linstor-csi creates one per PVC. +// +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.layerStack) || size(oldSelf.layerStack) == 0 || (has(self.layerStack) && size(self.layerStack) > 0)",message="layerStack is decided once: it may be set or replaced, never emptied" type ResourceDefinitionSpec struct { // externalName is the user-facing name surfaced by csi (CSI volume id). // Empty means the same as metadata.name. @@ -73,8 +75,10 @@ type ResourceDefinitionSpec struct { // Order is top-down: the first layer's device is what the // consumer Pod mounts, the last is the raw block device the // storage provider creates. - // Empty = inherits from the parent ResourceGroup; both empty = - // `["DRBD","STORAGE"]`. + // Empty means not decided yet: the controller fixes it once, from + // the parent ResourceGroup's stack for a definition that holds no + // data, and the satellite builds nothing until it is set. Once set it + // is never emptied. See docs/layer-stack.md. // +optional LayerStack []string `json:"layerStack,omitempty"` diff --git a/cmd/controller/main.go b/cmd/controller/main.go index aa8124e4..cbe48931 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -241,9 +241,10 @@ func main() { os.Exit(1) } if err := (&controller.ResourceDefinitionReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Store: st, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Store: st, + Recorder: mgr.GetEventRecorder("blockstor-controller"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "resourcedefinition") os.Exit(1) diff --git a/config/crd/bases/blockstor.cozystack.io_resourcedefinitions.yaml b/config/crd/bases/blockstor.cozystack.io_resourcedefinitions.yaml index 0bf82849..cf3fdc2a 100644 --- a/config/crd/bases/blockstor.cozystack.io_resourcedefinitions.yaml +++ b/config/crd/bases/blockstor.cozystack.io_resourcedefinitions.yaml @@ -343,8 +343,10 @@ spec: Order is top-down: the first layer's device is what the consumer Pod mounts, the last is the raw block device the storage provider creates. - Empty = inherits from the parent ResourceGroup; both empty = - `["DRBD","STORAGE"]`. + Empty means not decided yet: the controller fixes it once, from + the parent ResourceGroup's stack for a definition that holds no + data, and the satellite builds nothing until it is set. Once set it + is never emptied. See docs/layer-stack.md. items: type: string type: array @@ -472,6 +474,11 @@ spec: - volumeNumber x-kubernetes-list-type: map type: object + x-kubernetes-validations: + - message: 'layerStack is decided once: it may be set or replaced, never + emptied' + rule: '!has(oldSelf.layerStack) || size(oldSelf.layerStack) == 0 || + (has(self.layerStack) && size(self.layerStack) > 0)' status: description: status defines the observed state of ResourceDefinition properties: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 881cf2ac..866ed1b9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -61,3 +61,10 @@ rules: - get - patch - update +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch diff --git a/internal/controller/export_test.go b/internal/controller/export_test.go index e9d66c6a..eaa8e9c9 100644 --- a/internal/controller/export_test.go +++ b/internal/controller/export_test.go @@ -68,14 +68,6 @@ func AlreadyExists(err error) bool { return alreadyExists(err) } -// ResolveLayerStack exposes the four-tier layer-stack resolver: -// RD spec → RG spec → nil (dispatcher default). Tests pin each -// tier so a refactor that swapped precedence (RG over RD, say) -// would surface immediately. -func (r *ResourceReconciler) ResolveLayerStack(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) []string { - return r.resolveLayerStack(ctx, rd) -} - // ResolveEffectiveProps exposes the four-tier prop resolver: cluster // ControllerProps → RG → RD → Resource. Tests pin each tier and the // soft-fail-on-missing-RG path. @@ -248,3 +240,6 @@ const AutoTiebreakerSuppressedUntilAnnotation = apiv1.AutoTiebreakerSuppressedUn // annotation key for tests. Same compile-time-rename guard as the // suppression key above. const KeepTiebreakerUntilAnnotation = apiv1.KeepTiebreakerUntilAnnotation + +// RDReconcileRequeue exposes the RD reconciler's periodic requeue. +const RDReconcileRequeue = rdReconcileRequeue diff --git a/internal/controller/layer_stack_test.go b/internal/controller/layer_stack_test.go deleted file mode 100644 index c7b41410..00000000 --- a/internal/controller/layer_stack_test.go +++ /dev/null @@ -1,155 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -/* -Copyright 2026 Cozystack contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller_test - -import ( - "context" - "slices" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" - controllerpkg "github.com/cozystack/blockstor/internal/controller" -) - -// TestResolveLayerStackRDWins: when an RD specifies its own -// LayerStack, that wins over the parent RG's. Pins the precedence -// the dispatcher's CSI pass-through relies on (linstor-csi sets -// layer_list on autoplace; the REST handler persists it onto rd -// spec, and the resolver here picks it up unchanged). -func TestResolveLayerStackRDWins(t *testing.T) { - t.Parallel() - - scheme := newScheme(t) - - rg := &blockstoriov1alpha1.ResourceGroup{ - ObjectMeta: metav1.ObjectMeta{Name: "rg"}, - Spec: blockstoriov1alpha1.ResourceGroupSpec{ - SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{ - LayerStack: []string{"DRBD", "STORAGE"}, - }, - }, - } - - cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rg).Build() - rec := &controllerpkg.ResourceReconciler{Client: cli, Scheme: scheme} - - rd := &blockstoriov1alpha1.ResourceDefinition{ - Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ - ResourceGroupName: "rg", - LayerStack: []string{"LUKS", "STORAGE"}, - }, - } - - got := rec.ResolveLayerStack(context.Background(), rd) - if !slices.Equal(got, []string{"LUKS", "STORAGE"}) { - t.Errorf("RD wins over RG: got %v, want [LUKS STORAGE]", got) - } -} - -// TestResolveLayerStackFallsBackToRG: RD with no LayerStack inherits -// from its parent RG. This is the path operators use to set defaults -// (e.g. `linstor rg create encrypted-rg --layer-list LUKS,STORAGE`). -func TestResolveLayerStackFallsBackToRG(t *testing.T) { - t.Parallel() - - scheme := newScheme(t) - - rg := &blockstoriov1alpha1.ResourceGroup{ - ObjectMeta: metav1.ObjectMeta{Name: "rg-luks"}, - Spec: blockstoriov1alpha1.ResourceGroupSpec{ - SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{ - LayerStack: []string{"DRBD", "LUKS", "STORAGE"}, - }, - }, - } - - cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(rg).Build() - rec := &controllerpkg.ResourceReconciler{Client: cli, Scheme: scheme} - - rd := &blockstoriov1alpha1.ResourceDefinition{ - Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ - ResourceGroupName: "rg-luks", - // LayerStack intentionally absent. - }, - } - - got := rec.ResolveLayerStack(context.Background(), rd) - if !slices.Equal(got, []string{"DRBD", "LUKS", "STORAGE"}) { - t.Errorf("inherit from RG: got %v, want [DRBD LUKS STORAGE]", got) - } -} - -// TestResolveLayerStackNilWithoutRG: RD with no LayerStack and no -// ResourceGroupName → nil (dispatcher's default-fall-through). -// Pins the legacy single-RD path that pre-Phase-9 clients rely on. -func TestResolveLayerStackNilWithoutRG(t *testing.T) { - t.Parallel() - - scheme := newScheme(t) - cli := fake.NewClientBuilder().WithScheme(scheme).Build() - rec := &controllerpkg.ResourceReconciler{Client: cli, Scheme: scheme} - - rd := &blockstoriov1alpha1.ResourceDefinition{ - Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ - // No LayerStack, no ResourceGroupName. - }, - } - - if got := rec.ResolveLayerStack(context.Background(), rd); got != nil { - t.Errorf("nil-fallback: got %v, want nil", got) - } -} - -// TestResolveLayerStackMissingRG: RD points at an RG that doesn't -// exist (deleted under us, race) → nil rather than blowing up. The -// dispatcher then falls through to the satellite default. -func TestResolveLayerStackMissingRG(t *testing.T) { - t.Parallel() - - scheme := newScheme(t) - cli := fake.NewClientBuilder().WithScheme(scheme).Build() - rec := &controllerpkg.ResourceReconciler{Client: cli, Scheme: scheme} - - rd := &blockstoriov1alpha1.ResourceDefinition{ - Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ - ResourceGroupName: "ghost-rg", - }, - } - - if got := rec.ResolveLayerStack(context.Background(), rd); got != nil { - t.Errorf("missing-RG: got %v, want nil (no panic, no error)", got) - } -} - -// TestResolveLayerStackNilRD: defensive — handing a nil RD must yield -// nil, not panic. -func TestResolveLayerStackNilRD(t *testing.T) { - t.Parallel() - - scheme := newScheme(t) - cli := fake.NewClientBuilder().WithScheme(scheme).Build() - rec := &controllerpkg.ResourceReconciler{Client: cli, Scheme: scheme} - - if got := rec.ResolveLayerStack(context.Background(), nil); got != nil { - t.Errorf("nil RD: got %v, want nil", got) - } -} diff --git a/internal/controller/resource_controller.go b/internal/controller/resource_controller.go index c3ce3cb9..53a0597b 100644 --- a/internal/controller/resource_controller.go +++ b/internal/controller/resource_controller.go @@ -2075,32 +2075,6 @@ func (r *ResourceReconciler) resolveEffectiveProps(ctx context.Context, target * return effectiveprops.Resolve(ctx, r.Client, target, rdPtr) } -// resolveLayerStack walks the RD → RG hierarchy and returns the -// effective layer composition. Returns nil for the dispatcher's -// default-fall-through behaviour when nothing is set anywhere — the -// dispatcher then defaults to whatever rd.Spec.LayerStack contains -// (also possibly nil → satellite-side default ["DRBD","STORAGE"]). -func (r *ResourceReconciler) resolveLayerStack(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) []string { - if rd == nil { - return nil - } - - if len(rd.Spec.LayerStack) > 0 { - return rd.Spec.LayerStack - } - - if rd.Spec.ResourceGroupName == "" { - return nil - } - - var rg blockstoriov1alpha1.ResourceGroup - if err := r.Get(ctx, client.ObjectKey{Name: rd.Spec.ResourceGroupName}, &rg); err != nil { - return nil - } - - return rg.Spec.SelectFilter.LayerStack -} - // EnsureDRBDIDsForTest is an exported alias for ensureDRBDIDs. The // allocator is package-private because it's an internal reconciler // step, but the property tests live in package controller_test and diff --git a/internal/controller/resourcedefinition_controller.go b/internal/controller/resourcedefinition_controller.go index ba8ff36d..a0fc56da 100644 --- a/internal/controller/resourcedefinition_controller.go +++ b/internal/controller/resourcedefinition_controller.go @@ -30,6 +30,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -86,6 +87,11 @@ type ResourceDefinitionReconciler struct { // reconciler directly and skip this — the field is nil-safe // and falls back to the cached client below. APIReader client.Reader + + // Recorder emits the warning Event for an RD that kept the default + // layer stack because it already held data. Nil in tests that do + // not assert on Events. + Recorder events.EventRecorder } // +kubebuilder:rbac:groups=blockstor.cozystack.io,resources=resourcedefinitions,verbs=get;list;watch;create;update;patch;delete @@ -120,6 +126,13 @@ func (r *ResourceDefinitionReconciler) Reconcile(ctx context.Context, req ctrl.R return ctrl.Result{}, nil } + err = r.pinLayerStack(ctx, &rd) + if err != nil { + log.Error(err, "pin layer stack", "rd", rd.Name) + + return ctrl.Result{}, err + } + err = r.ensureTiebreaker(ctx, &rd) if err != nil { log.Error(err, "ensure tiebreaker", "rd", rd.Name) @@ -941,10 +954,10 @@ func (r *ResourceDefinitionReconciler) rollbackWitnessIfRDGone(ctx context.Conte } // resolveRDLayerStack returns the effective layer composition for -// the RD by walking RD → RG → default. Mirrors the (unexported) -// ResourceReconciler.resolveLayerStack but lives on the RD -// reconciler so the witness gate (Bug 334) doesn't have to depend on -// the resource-reconciler instance. +// the RD by walking RD → RG → default, for the witness gate (Bug 334). +// It runs before pinLayerStack has written the group's stack onto a +// fresh RD, so it reads the group itself; it decides only whether a +// witness is placed, never what goes to disk. // // Read order: // diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go new file mode 100644 index 00000000..d2069c0c --- /dev/null +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "slices" + "strings" + + "github.com/cockroachdb/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// layerStackNotInheritedReason is the Event reason for an RD that kept +// the default stack because it already held data under it. +const layerStackNotInheritedReason = "LayerStackNotInherited" + +// layerStackPinAction is the Event action: the controller fixing the +// RD's layer stack. +const layerStackPinAction = "PinLayerStack" + +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch + +// pinLayerStack writes the layer stack onto an RD that leaves it to its +// resource group. The stack is decided once, here, and every later +// reader takes it from the RD: the satellite waits for this pin rather +// than resolve the group itself, so a change to the group never +// reshapes a volume that already exists. +// +// An RD that holds no data yet takes the group's stack. One that does +// already went to disk under the default stack; giving it the group's +// now would have the satellite build a different stack over that data +// (luksFormat over a plaintext volume). It is pinned to the stack it +// actually has, the group's stack is recorded in an annotation, and a +// warning Event names it, so the volume reads as what it is. +func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) error { + if len(rd.Spec.LayerStack) > 0 || rd.Spec.ResourceGroupName == "" { + return nil + } + + reader := r.directOrCached() + + var rg blockstoriov1alpha1.ResourceGroup + + // The RD keeps the group's name as the operator spelled it; the CR is + // stored under the folded one. + err := reader.Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rg) + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return errors.Wrapf(err, "read resource group %q", rd.Spec.ResourceGroupName) + } + + groupStack := rg.Spec.SelectFilter.LayerStack + if len(groupStack) == 0 { + return nil + } + + holdsData, err := r.rdHoldsData(ctx, rd.Name) + if err != nil { + return err + } + + stack := groupStack + if holdsData { + stack = apiv1.DefaultLayerStack() + } + + base := rd.DeepCopy() + rd.Spec.LayerStack = slices.Clone(stack) + + unapplied := !slices.Equal(stack, groupStack) + if unapplied { + if rd.Annotations == nil { + rd.Annotations = map[string]string{} + } + + rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] = strings.Join(groupStack, ",") + } + + err = r.Patch(ctx, rd, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + if err != nil { + return errors.Wrapf(err, "pin layer stack of %q", rd.Name) + } + + logf.FromContext(ctx).Info("pinned layer stack", "rd", rd.Name, "stack", stack, "group", rg.Name, + "groupStack", groupStack, "groupStackApplied", !unapplied) + + if unapplied && r.Recorder != nil { + r.Recorder.Eventf(rd, nil, corev1.EventTypeWarning, layerStackNotInheritedReason, layerStackPinAction, + "already holds data under layer stack %s, so the stack %s of resource group %s was not applied; migrate the data to a new volume to take it", + strings.Join(stack, ","), strings.Join(groupStack, ","), rg.Name) + } + + return nil +} + +// rdHoldsData reports whether any replica of the RD has gone to disk: +// DRBD metadata created, or a volume the satellite reported a device +// for. Either means the storage under it was built with the stack the +// RD carried then. +func (r *ResourceDefinitionReconciler) rdHoldsData(ctx context.Context, rdName string) (bool, error) { + var list blockstoriov1alpha1.ResourceList + + err := r.directOrCached().List(ctx, &list) + if err != nil { + return false, errors.Wrapf(err, "list replicas of %q", rdName) + } + + for i := range list.Items { + res := &list.Items[i] + if res.Spec.ResourceDefinitionName != rdName { + continue + } + + if meta.IsStatusConditionTrue(res.Status.Conditions, blockstoriov1alpha1.ConditionMetadataCreated) { + return true, nil + } + + for j := range res.Status.Volumes { + if res.Status.Volumes[j].DevicePath != "" { + return true, nil + } + } + } + + return false, nil +} diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go new file mode 100644 index 00000000..7aeba1d6 --- /dev/null +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller_test + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + controllerpkg "github.com/cozystack/blockstor/internal/controller" + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var ( + luksStack = []string{"DRBD", "LUKS", "STORAGE"} + + errRead = errors.New("apiserver unavailable") +) + +func pinGroup(stack ...string) *blockstoriov1alpha1.ResourceGroup { + return &blockstoriov1alpha1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "rg-pin"}, + Spec: blockstoriov1alpha1.ResourceGroupSpec{ + SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{LayerStack: stack}, + }, + } +} + +func pinRD(group string, stack ...string) *blockstoriov1alpha1.ResourceDefinition { + return &blockstoriov1alpha1.ResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-pin"}, + Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ResourceGroupName: group, LayerStack: stack}, + } +} + +func pinReplica(rd, node string) *blockstoriov1alpha1.Resource { + return &blockstoriov1alpha1.Resource{ + ObjectMeta: metav1.ObjectMeta{Name: rd + "-" + node}, + Spec: blockstoriov1alpha1.ResourceSpec{ResourceDefinitionName: rd, NodeName: node}, + } +} + +// withMetadata marks a replica whose DRBD metadata was created. +func withMetadata(res *blockstoriov1alpha1.Resource) *blockstoriov1alpha1.Resource { + res.Status.Conditions = []metav1.Condition{{ + Type: blockstoriov1alpha1.ConditionMetadataCreated, Status: metav1.ConditionTrue, + Reason: "Created", LastTransitionTime: metav1.Now(), + }} + + return res +} + +// withDevice marks a replica whose satellite reported a volume device. +func withDevice(res *blockstoriov1alpha1.Resource) *blockstoriov1alpha1.Resource { + res.Status.Volumes = []blockstoriov1alpha1.ResourceVolumeStatus{{VolumeNumber: 0, DevicePath: "/dev/drbd1000"}} + + return res +} + +type pinResult struct { + result ctrl.Result + rd blockstoriov1alpha1.ResourceDefinition + events []string + err error +} + +func reconcilePin(t *testing.T, cli client.Client) pinResult { + t.Helper() + + recorder := events.NewFakeRecorder(10) + rec := &controllerpkg.ResourceDefinitionReconciler{ + Client: cli, + Scheme: newScheme(t), + Store: store.NewInMemory(), + Recorder: recorder, + } + + result, err := rec.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Name: "pvc-pin"}}) + + var out pinResult + + out.result, out.err = result, err + + getErr := cli.Get(context.Background(), client.ObjectKey{Name: "pvc-pin"}, &out.rd) + if getErr != nil { + t.Fatalf("read RD back: %v", getErr) + } + + close(recorder.Events) + + for e := range recorder.Events { + out.events = append(out.events, e) + } + + return out +} + +func pinClient(t *testing.T, objs ...client.Object) client.WithWatch { + t.Helper() + + return fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(objs...).Build() +} + +// An RD with nothing on disk yet takes its group's stack, LUKS +// included, so the satellite builds it encrypted from the start. +func TestPinLayerStackGivesAFreshRDItsGroupsStack(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup(luksStack...), pinRD("rg-pin"), pinReplica("pvc-pin", "n1"))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, luksStack) { + t.Fatalf("stack %v, want %v", got.rd.Spec.LayerStack, luksStack) + } + + if _, ok := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; ok { + t.Error("a fresh RD carries the unapplied-stack annotation") + } + + if len(got.events) != 0 { + t.Errorf("events %v, want none", got.events) + } +} + +// Nothing watches a group's stack on an RD's behalf: an RD whose group +// has no stack yet picks one up only on its next reconcile, which the +// controller schedules every time. +func TestPinLayerStackComesBackForAGroupWithoutAStack(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup(), pinRD("rg-pin"))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if got.result.RequeueAfter != controllerpkg.RDReconcileRequeue { + t.Errorf("requeue after %v, want %v", got.result.RequeueAfter, controllerpkg.RDReconcileRequeue) + } +} + +// LINSTOR names are case-insensitive. An RD that names its group as the +// operator spelled it (`--resource-group RG-PIN`) is pinned from the +// group's CR, which is stored under the folded name. +func TestPinLayerStackFindsAGroupNamedInAnotherCase(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup(luksStack...), pinRD("RG-Pin"), pinReplica("pvc-pin", "n1"))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, luksStack) { + t.Fatalf("stack %v, want the group's %v", got.rd.Spec.LayerStack, luksStack) + } +} + +// An RD that already went to disk under the default stack keeps it: +// the group's LUKS stack over that data would luksFormat it. The RD +// says what it is, and the operator is told. +func TestPinLayerStackKeepsTheAppliedStackOfAnRDThatHoldsData(t *testing.T) { + t.Parallel() + + for name, replica := range map[string]*blockstoriov1alpha1.Resource{ + "metadata created": withMetadata(pinReplica("pvc-pin", "n1")), + "device reported": withDevice(pinReplica("pvc-pin", "n1")), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup(luksStack...), pinRD("rg-pin"), replica, pinReplica("pvc-pin", "n2"))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, apiv1.DefaultLayerStack()) { + t.Fatalf("stack %v, want the applied default %v", got.rd.Spec.LayerStack, apiv1.DefaultLayerStack()) + } + + if a := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; a != "DRBD,LUKS,STORAGE" { + t.Errorf("annotation %q, want the group's stack", a) + } + + if len(got.events) != 1 || !strings.HasPrefix(got.events[0], "Warning LayerStackNotInherited") || + !strings.Contains(got.events[0], "DRBD,LUKS,STORAGE") { + t.Errorf("events %v, want one LayerStackNotInherited warning naming the group's stack", got.events) + } + }) + } +} + +// Data under the default stack in a group whose stack is the default +// is no conflict: pinned as is, without a warning. +func TestPinLayerStackDataUnderTheGroupsOwnStackIsNoConflict(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup("DRBD", "STORAGE"), pinRD("rg-pin"), withMetadata(pinReplica("pvc-pin", "n1")))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, []string{"DRBD", "STORAGE"}) { + t.Fatalf("stack %v", got.rd.Spec.LayerStack) + } + + if _, ok := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; ok || len(got.events) != 0 { + t.Errorf("annotations %v, events %v, want neither", got.rd.Annotations, got.events) + } +} + +// Data on another RD's replica says nothing about this one. +func TestPinLayerStackLooksOnlyAtItsOwnReplicas(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup(luksStack...), pinRD("rg-pin"), withMetadata(pinReplica("pvc-other", "n1")))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, luksStack) { + t.Fatalf("stack %v, want %v", got.rd.Spec.LayerStack, luksStack) + } +} + +// Every shape that has nothing to inherit is left exactly as it is. +func TestPinLayerStackLeavesRDsWithNothingToInherit(t *testing.T) { + t.Parallel() + + cases := map[string][]client.Object{ + "RD sets its own stack": {pinGroup(luksStack...), pinRD("rg-pin", "DRBD", "STORAGE")}, + "RD in no group": {pinGroup(luksStack...), pinRD("")}, + "group gone": {pinRD("rg-pin")}, + "group sets no stack": {pinGroup(), pinRD("rg-pin")}, + } + + for name, objs := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + var before blockstoriov1alpha1.ResourceDefinition + + patched := false + cli := interceptor.NewClient(pinClient(t, objs...), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*blockstoriov1alpha1.ResourceDefinition); ok { + patched = true + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }) + if err := cli.Get(context.Background(), client.ObjectKey{Name: "pvc-pin"}, &before); err != nil { + t.Fatalf("read RD: %v", err) + } + + got := reconcilePin(t, cli) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, before.Spec.LayerStack) || len(got.rd.Annotations) != 0 { + t.Errorf("stack %v annotations %v, want %v and none", got.rd.Spec.LayerStack, got.rd.Annotations, before.Spec.LayerStack) + } + + if patched { + t.Error("the RD was written although there was nothing to pin") + } + }) + } +} + +// Neither an unreadable group nor an unreadable replica list is a +// licence to pin: either could hide the answer. +func TestPinLayerStackFailsClosedOnAReadError(t *testing.T) { + t.Parallel() + + for name, funcs := range map[string]interceptor.Funcs{ + "group": {Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*blockstoriov1alpha1.ResourceGroup); ok { + return errRead + } + + return c.Get(ctx, key, obj, opts...) + }}, + "replicas": {List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*blockstoriov1alpha1.ResourceList); ok { + return errRead + } + + return c.List(ctx, list, opts...) + }}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + cli := interceptor.NewClient(pinClient(t, pinGroup(luksStack...), pinRD("rg-pin")), funcs) + + got := reconcilePin(t, cli) + if !errors.Is(got.err, errRead) { + t.Fatalf("reconcile: %v, want the read error", got.err) + } + + if len(got.rd.Spec.LayerStack) != 0 { + t.Errorf("stack %v pinned on a read error", got.rd.Spec.LayerStack) + } + }) + } +} + +// The pin only fills an empty stack. A writer that set one first (the +// REST autoplace stamp) wins; the pin must not overwrite it from a +// stale copy. +func TestPinLayerStackDoesNotOverwriteARacingWriter(t *testing.T) { + t.Parallel() + + base := pinClient(t, pinGroup(luksStack...), pinRD("rg-pin")) + raced := false + + cli := interceptor.NewClient(base, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if !raced { + raced = true + + var live blockstoriov1alpha1.ResourceDefinition + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), &live); err != nil { + return err + } + + live.Spec.LayerStack = []string{"DRBD", "STORAGE"} + if err := c.Update(ctx, &live); err != nil { + return err + } + } + + return c.Patch(ctx, obj, patch, opts...) + }, + }) + + got := reconcilePin(t, cli) + if got.err == nil { + t.Fatal("reconcile succeeded over a racing writer") + } + + if !slices.Equal(got.rd.Spec.LayerStack, []string{"DRBD", "STORAGE"}) { + t.Fatalf("stack %v, want the racing writer's", got.rd.Spec.LayerStack) + } +} diff --git a/pkg/api/v1/annotations.go b/pkg/api/v1/annotations.go index fa1e193c..3431f525 100644 --- a/pkg/api/v1/annotations.go +++ b/pkg/api/v1/annotations.go @@ -140,3 +140,12 @@ const PeerChangedAnnotation = "blockstor.io/peer-changed" // truth without either package importing the other — pkg/api/v1 is // the neutral, dependency-free shared layer both already import. const RDSpawnShortfallAnnotation = "blockstor.io/spawn-shortfall" + +// UnappliedGroupLayerStackAnnotation is stamped on a ResourceDefinition +// whose layer stack was left to its resource group but which already +// held data under the default stack when the controller came to fix +// it. The controller pins the stack the satellite actually applied and +// records here, comma-separated, the group's stack that was not, so an +// operator can find every volume that is not what its group promises +// (for example a plaintext volume in a LUKS group) and migrate it. +const UnappliedGroupLayerStackAnnotation = "blockstor.io/unapplied-group-layer-stack" diff --git a/pkg/store/inmemory_resource_definition.go b/pkg/store/inmemory_resource_definition.go index 06234153..767873ec 100644 --- a/pkg/store/inmemory_resource_definition.go +++ b/pkg/store/inmemory_resource_definition.go @@ -95,6 +95,11 @@ func (s *inMemoryResourceDefinitions) Update(_ context.Context, rd *apiv1.Resour } next := *rd + // A decided layer stack is never emptied by a write-back of an older + // copy, as on the k8s store. + if len(next.LayerStack) == 0 { + next.LayerStack = prev.LayerStack + } // Bug-021: nil wire annotations = "untouched"; empty = "clear". // Mirrors the k8s store's mergeUserAnnotationsInto contract. next.Annotations = carryAnnotationsOnNil(next.Annotations, prev.Annotations) diff --git a/pkg/store/k8s/layer_stack_test.go b/pkg/store/k8s/layer_stack_test.go new file mode 100644 index 00000000..4abcb762 --- /dev/null +++ b/pkg/store/k8s/layer_stack_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s_test + +import ( + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + crdv1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" +) + +// TestTheAPIServerRefusesToEmptyADecidedLayerStack: the store keeps a +// pinned stack across a stale write-back, and the CRD's transition rule +// holds the same line for every other writer (kubectl, another +// controller). Setting a stack and replacing it stay allowed. +func TestTheAPIServerRefusesToEmptyADecidedLayerStack(t *testing.T) { + if fixture == nil { + t.Skip("envtest assets not installed; run `make setup-envtest` to enable") + } + + t.Cleanup(func() { wipeAll(t, fixture.client) }) + + ctx := t.Context() + rd := &crdv1alpha1.ResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "pvc-cel-pin"}} + + if err := fixture.client.Create(ctx, rd); err != nil { + t.Fatalf("create with no stack: %v", err) + } + + rd.Spec.LayerStack = []string{"DRBD", "LUKS", "STORAGE"} + if err := fixture.client.Update(ctx, rd); err != nil { + t.Fatalf("set the stack: %v", err) + } + + rd.Spec.LayerStack = []string{"DRBD", "STORAGE"} + if err := fixture.client.Update(ctx, rd); err != nil { + t.Fatalf("replace the stack: %v", err) + } + + rd.Spec.LayerStack = nil + + err := fixture.client.Update(ctx, rd) + if err == nil { + t.Fatal("emptying a decided stack: want the transition rule's refusal, got nil") + } + + if !strings.Contains(err.Error(), "never emptied") { + t.Fatalf("emptying a decided stack: want the transition rule's message, got %v", err) + } +} diff --git a/pkg/store/k8s/resource_definitions.go b/pkg/store/k8s/resource_definitions.go index 1b8a9984..3a0fe9b2 100644 --- a/pkg/store/k8s/resource_definitions.go +++ b/pkg/store/k8s/resource_definitions.go @@ -163,7 +163,17 @@ func (s *resourceDefinitions) Update(ctx context.Context, in *apiv1.ResourceDefi // naïve rebuild would wipe it and a later replica added to this // established RD would wrongly skip the initial sync. prevInitialized := existing.Spec.Initialized + // The layer stack is decided once and then only replaced, never + // emptied: a caller writing back a copy read before the pin would + // otherwise undo it, and the RD reconcile would then judge the + // volume built under the default and re-pin that over LUKS data. + prevLayerStack := existing.Spec.LayerStack existing.Spec = wireToCRDRDSpec(in) + + if len(existing.Spec.LayerStack) == 0 { + existing.Spec.LayerStack = prevLayerStack + } + existing.Spec.VolumeDefinitions = prevVDs existing.Spec.Encryption = prevEncryption // Identity-to-spec: preserve the optional controller/operator diff --git a/pkg/store/storetest/storetest.go b/pkg/store/storetest/storetest.go index 5beedf4a..ecfc1a1a 100644 --- a/pkg/store/storetest/storetest.go +++ b/pkg/store/storetest/storetest.go @@ -1282,6 +1282,7 @@ func RunResourceDefinitionStore(t *testing.T, newStore Factory) { // Bug-021: nil = untouched / non-nil = replace / empty = clear. // See annotation_contract.go. t.Run("UpdateAnnotationContract", func(t *testing.T) { testRDUpdateAnnotationContract(t, newStore) }) + t.Run("UpdateKeepsADecidedLayerStack", func(t *testing.T) { testRDUpdateKeepsADecidedLayerStack(t, newStore) }) t.Run("UpdateChangesProps", func(t *testing.T) { s := newStore(t).ResourceDefinitions() ctx := t.Context() @@ -1309,6 +1310,54 @@ func RunResourceDefinitionStore(t *testing.T, newStore Factory) { }) } +// testRDUpdateKeepsADecidedLayerStack: a write-back of a copy read +// before the layer stack was pinned carries no stack. It must leave +// the pin in place, or the RD reconcile would decide again over a +// volume that may already hold data. Replacing a pinned stack with +// another non-empty one still goes through. +func testRDUpdateKeepsADecidedLayerStack(t *testing.T, newStore Factory) { + t.Helper() + s := newStore(t).ResourceDefinitions() + ctx := t.Context() + if err := s.Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-pin"}); err != nil { + t.Fatalf("Create: %v", err) + } + stale, err := s.Get(ctx, "pvc-pin") + if err != nil { + t.Fatalf("Get: %v", err) + } + pinned := stale + pinned.LayerStack = []string{"DRBD", "LUKS", "STORAGE"} + if err := s.Update(ctx, &pinned); err != nil { + t.Fatalf("Update (pin): %v", err) + } + stale.Props = map[string]string{"k": "v"} + if err := s.Update(ctx, &stale); err != nil { + t.Fatalf("Update (stale write-back): %v", err) + } + got, err := s.Get(ctx, "pvc-pin") + if err != nil { + t.Fatalf("Get: %v", err) + } + if !slices.Equal(got.LayerStack, []string{"DRBD", "LUKS", "STORAGE"}) { + t.Errorf("LayerStack after a stale write-back: got %v, want [DRBD LUKS STORAGE]", got.LayerStack) + } + if got.Props["k"] != "v" { + t.Errorf("Props: got %v, want the write-back's k=v", got.Props) + } + got.LayerStack = []string{"DRBD", "STORAGE"} + if err := s.Update(ctx, &got); err != nil { + t.Fatalf("Update (replace): %v", err) + } + got, err = s.Get(ctx, "pvc-pin") + if err != nil { + t.Fatalf("Get: %v", err) + } + if !slices.Equal(got.LayerStack, []string{"DRBD", "STORAGE"}) { + t.Errorf("LayerStack after a replace: got %v, want [DRBD STORAGE]", got.LayerStack) + } +} + // RunResourceGroupStore exercises every branch of store.ResourceGroupStore. func RunResourceGroupStore(t *testing.T, newStore Factory) { t.Helper() From 26af89ddf23ae86d602163aa6316e4c1cd7bbdda Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 12 Oct 2026 23:55:24 +0200 Subject: [PATCH 02/22] fix(satellite): build no volume before its layer stack is pinned The satellite read a definition's empty layer stack as the default and built the volume from it, even when the definition's group sets another stack. It now waits, on a short requeue, until the controller has written the group's stack onto the definition, and the intent builder refuses such a definition outright, so no path turns the empty stack into storage. The group is looked up under its folded name, so a definition naming it in another case waits too rather than being taken for one outside any group. A definition in no group, or in a group without a stack, keeps the default as before; a group that cannot be read applies nothing. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/satellite/controllers/layer_stack_pin.go | 79 ++++++ .../layer_stack_pin_internal_test.go | 224 ++++++++++++++++++ pkg/satellite/controllers/resource.go | 20 +- 3 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 pkg/satellite/controllers/layer_stack_pin.go create mode 100644 pkg/satellite/controllers/layer_stack_pin_internal_test.go diff --git a/pkg/satellite/controllers/layer_stack_pin.go b/pkg/satellite/controllers/layer_stack_pin.go new file mode 100644 index 00000000..2f0894c8 --- /dev/null +++ b/pkg/satellite/controllers/layer_stack_pin.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "time" + + "github.com/cockroachdb/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// errLayerStackNotPinned reports an RD that leaves its layer stack to its +// resource group, which the controller has not yet written onto it. +var errLayerStackNotPinned = errors.New("the resource definition's layer stack comes from its group and is not pinned yet") + +// layerStackPinRequeue is how soon a replica waiting on the pin is tried +// again; the controller pins on the definition's own reconcile. +const layerStackPinRequeue = 2 * time.Second + +// checkLayerStackPinned refuses an RD whose layer stack is empty while +// its resource group sets one. +// +// Every reader on the data path (the intent, the DRBD gate, the LUKS +// passphrase) reads rd.Spec.LayerStack, and an empty one there means the +// default stack, which has no LUKS. An RD created without a stack, by +// kubectl or GitOps or a door that does not copy the group's, would then +// go to disk unencrypted while the gates that resolve through the group +// called it encrypted. The stack is fixed once, by the controller, which +// knows whether the definition already holds data under the default stack +// (ResourceDefinitionReconciler.pinLayerStack); the satellite waits for +// that rather than resolve it a second time and disagree. +// +// A group that is gone, or sets no stack, leaves the default in force, as +// before. A group that cannot be read fails closed. +func (r *ResourceReconciler) checkLayerStackPinned(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) error { + if len(rd.Spec.LayerStack) > 0 || rd.Spec.ResourceGroupName == "" { + return nil + } + + var rg blockstoriov1alpha1.ResourceGroup + + // The RD keeps the group's name as the operator spelled it; the CR is + // stored under the folded one. + err := r.Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rg) + if apierrors.IsNotFound(err) { + return nil + } + + if err != nil { + return errors.Wrapf(err, "read resource group %q for the layer stack of %q", rd.Spec.ResourceGroupName, rd.Name) + } + + if len(rg.Spec.SelectFilter.LayerStack) == 0 { + return nil + } + + return errors.Wrapf(errLayerStackNotPinned, "%q in group %q", rd.Name, rd.Spec.ResourceGroupName) +} diff --git a/pkg/satellite/controllers/layer_stack_pin_internal_test.go b/pkg/satellite/controllers/layer_stack_pin_internal_test.go new file mode 100644 index 00000000..61507d41 --- /dev/null +++ b/pkg/satellite/controllers/layer_stack_pin_internal_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "errors" + "slices" + "sync/atomic" + "testing" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + "github.com/cozystack/blockstor/pkg/satellite" + "github.com/cozystack/blockstor/pkg/storage" +) + +var errGroupRead = errors.New("apiserver unavailable") + +// luksGroup is a resource group whose layer stack carries LUKS. +func luksGroup() *blockstoriov1alpha1.ResourceGroup { + return &blockstoriov1alpha1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "rg-luks"}, + Spec: blockstoriov1alpha1.ResourceGroupSpec{ + SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}, + }, + } +} + +// groupMemberRD is an RD in rg-luks that sets no layer stack of its own. +func groupMemberRD(stack ...string) *blockstoriov1alpha1.ResourceDefinition { + return &blockstoriov1alpha1.ResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-member"}, + Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ + ResourceGroupName: "rg-luks", + LayerStack: stack, + VolumeDefinitions: []blockstoriov1alpha1.ResourceDefinitionVolume{ + {VolumeNumber: 0, SizeKib: 4 * 1024}, + }, + }, + } +} + +func memberReplica() *blockstoriov1alpha1.Resource { + return &blockstoriov1alpha1.Resource{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-member-node-a"}, + Spec: blockstoriov1alpha1.ResourceSpec{ + ResourceDefinitionName: "pvc-member", + NodeName: "node-a", + StoragePool: "pool-a", + }, + } +} + +// An RD that leaves its layer stack to its group must not be built into +// an intent from its own empty stack: that intent has no LUKS layer and +// carries no passphrase, so the volume would go to disk unencrypted +// while everything else reads the group's stack and calls it encrypted. +func TestAnRDLeftToItsGroupsLUKSStackIsNotBuiltPlaintext(t *testing.T) { + t.Parallel() + + rd := groupMemberRD() + r := newPassphraseTestReconciler(t, luksGroup(), rd, memberReplica(), passphraseSecret("pass-member")) + + desired, err := r.buildDesiredFromCRD(t.Context(), memberReplica(), rd, nil) + if err == nil { + if !slices.Contains(desired.GetLayerStack(), "LUKS") || desired.GetProps()["LuksPassphrase"] == "" { + t.Fatalf("intent for an RD in a LUKS group: stack %v, passphrase set %v; want LUKS with its passphrase, or no intent", + desired.GetLayerStack(), desired.GetProps()["LuksPassphrase"] != "") + } + + return + } + + if !errors.Is(err, errLayerStackNotPinned) { + t.Fatalf("build: %v, want errLayerStackNotPinned", err) + } +} + +// Once the controller has written the group's stack onto the RD, the +// intent carries LUKS and the passphrase. +func TestAPinnedGroupStackBuildsLUKSWithItsPassphrase(t *testing.T) { + t.Parallel() + + rd := groupMemberRD("DRBD", "LUKS", "STORAGE") + r := newPassphraseTestReconciler(t, luksGroup(), rd, memberReplica(), passphraseSecret("pass-member")) + + desired, err := r.buildDesiredFromCRD(t.Context(), memberReplica(), rd, nil) + if err != nil { + t.Fatalf("build: %v", err) + } + + if !slices.Contains(desired.GetLayerStack(), "LUKS") { + t.Errorf("stack %v, want LUKS", desired.GetLayerStack()) + } + + if desired.GetProps()["LuksPassphrase"] != "pass-member" { + t.Error("the passphrase did not reach the intent") + } +} + +// The wait applies only to an RD whose group sets a stack it does not +// carry yet. Every other shape goes on with the stack it has. +func TestCheckLayerStackPinned(t *testing.T) { + t.Parallel() + + noStackGroup := luksGroup() + noStackGroup.Spec.SelectFilter.LayerStack = nil + + outsideGroup := groupMemberRD() + outsideGroup.Spec.ResourceGroupName = "" + + // LINSTOR names are case-insensitive: the RD keeps the operator's + // spelling, the group's CR is stored under the folded one. + mixedCase := groupMemberRD() + mixedCase.Spec.ResourceGroupName = "RG-LUKS" + + cases := []struct { + name string + rd *blockstoriov1alpha1.ResourceDefinition + objs []client.Object + want error + }{ + {name: "group stack not on the RD", rd: groupMemberRD(), objs: []client.Object{luksGroup()}, want: errLayerStackNotPinned}, + {name: "group named in another case", rd: mixedCase, objs: []client.Object{luksGroup()}, want: errLayerStackNotPinned}, + {name: "RD carries its stack", rd: groupMemberRD("DRBD", "STORAGE"), objs: []client.Object{luksGroup()}}, + {name: "RD in no group", rd: outsideGroup}, + {name: "group gone", rd: groupMemberRD()}, + {name: "group sets no stack", rd: groupMemberRD(), objs: []client.Object{noStackGroup}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + r := newPassphraseTestReconciler(t, tc.objs...) + + err := r.checkLayerStackPinned(t.Context(), tc.rd) + if tc.want == nil && err != nil { + t.Fatalf("check: %v, want nil", err) + } + + if tc.want != nil && !errors.Is(err, tc.want) { + t.Fatalf("check: %v, want %v", err, tc.want) + } + }) + } +} + +// A group that cannot be read is no licence to apply the default stack. +func TestCheckLayerStackPinnedFailsClosedOnAnUnreadableGroup(t *testing.T) { + t.Parallel() + + r := newPassphraseTestReconciler(t, luksGroup()) + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, obj client.Object, _ ...client.GetOption) error { + if _, ok := obj.(*blockstoriov1alpha1.ResourceGroup); ok { + return errGroupRead + } + + return nil + }, + }) + + err := r.checkLayerStackPinned(t.Context(), groupMemberRD()) + if !errors.Is(err, errGroupRead) || errors.Is(err, errLayerStackNotPinned) { + t.Fatalf("check: %v, want the read error", err) + } +} + +// The reconcile waits for the pin quietly, on a short requeue, rather +// than surfacing an error and backing off, and builds nothing meanwhile. +func TestRunApplyWaitsForTheLayerStackPin(t *testing.T) { + t.Parallel() + + rd := groupMemberRD() + res := memberReplica() + r := newPassphraseTestReconciler(t, luksGroup(), rd, res) + + provider := &flakyCreateProvider{} + exec := storage.NewFakeExec() + r.Config.Exec = exec + r.Config.Apply = satellite.NewReconciler(satellite.ReconcilerConfig{ + Providers: map[string]storage.Provider{"pool-a": provider}, + NodeName: "node-a", + }) + + result, err := r.runApply(t.Context(), res, logr.Discard()) + if err != nil { + t.Fatalf("runApply: %v", err) + } + + if result.RequeueAfter != layerStackPinRequeue { + t.Fatalf("requeue after %v, want %v", result.RequeueAfter, layerStackPinRequeue) + } + + if n := atomic.LoadInt32(&provider.created); n != 0 { + t.Errorf("%d volumes created before the pin, want none", n) + } + + if cmds := exec.CommandLines(); len(cmds) != 0 { + t.Errorf("commands run before the pin: %v, want none", cmds) + } +} diff --git a/pkg/satellite/controllers/resource.go b/pkg/satellite/controllers/resource.go index 3297b65a..6809d182 100644 --- a/pkg/satellite/controllers/resource.go +++ b/pkg/satellite/controllers/resource.go @@ -476,6 +476,17 @@ func (r *ResourceReconciler) runApply(ctx context.Context, res *blockstoriov1alp // election relies on (Bug 80). adoptAuthoritativeFlags(res, peers) + err = r.checkLayerStackPinned(ctx, &rd) + if errors.Is(err, errLayerStackNotPinned) { + logger.Info("waiting for the controller to pin the layer stack", "rd", rd.Name) + + return ctrl.Result{RequeueAfter: layerStackPinRequeue}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + // The DRBD-ID allocation gate only matters when the RD actually // uses DRBD. A LayerStack=["STORAGE"] (or ["LUKS","STORAGE"]) RD // renders no .res and the kernel never sees a node-id, so waiting @@ -1063,9 +1074,16 @@ func (r *ResourceReconciler) enqueueLocalSiblings(ctx context.Context, obj clien // same work the controller's dispatcher did, just on the // satellite side. Phase 10.1. func (r *ResourceReconciler) buildDesiredFromCRD(ctx context.Context, target *blockstoriov1alpha1.Resource, rd *blockstoriov1alpha1.ResourceDefinition, peers []blockstoriov1alpha1.Resource) (*intent.DesiredResource, error) { + // Checked again here, not only in runApply: no intent is ever built + // from a stack the definition does not carry yet. + err := r.checkLayerStackPinned(ctx, rd) + if err != nil { + return nil, err + } + var nodeList blockstoriov1alpha1.NodeList - err := r.List(ctx, &nodeList) + err = r.List(ctx, &nodeList) if err != nil { return nil, errors.Wrap(err, "list Nodes") } From d22ff1d42cc7910c266f15a59600cfca477ed4e3 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 13 Oct 2026 00:02:18 +0200 Subject: [PATCH 03/22] fix(luks): refuse to luksFormat a device that already holds data Format took "not a LUKS device" to mean "empty" and ran luksFormat. A volume that went to disk without LUKS and is later asked for it, by a layer stack that changed under it or a restore that brought data in, is not LUKS either, and luksFormat destroys it. Format now probes the device with `blkid -p` first and formats only one that carries no signature at all. DRBD internal metadata, a filesystem or a partition table refuses it with ErrDeviceHoldsData, naming what was found; a probe that cannot say the device is empty refuses it too. blkid exits 2 both for "nothing found" and for a device it could not open, so exit 2 counts as empty only without an "error:" line on stderr and for a path that stats as a block device; a missing path or a regular file is refused. The apply of that replica fails with the reason, and nothing is written. The satellite also records the refusal as a LUKSDeviceHoldsData warning Event on the replica, through events.k8s.io, for which its role gains the events rule. A signature left on reused extents is cleared with wipefs; the message says so. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- config/rbac/satellite_role.yaml | 9 + pkg/luks/luks.go | 128 ++++++++- pkg/luks/luks_bug_175_test.go | 12 +- pkg/luks/luks_holds_data_test.go | 253 ++++++++++++++++++ pkg/luks/luks_test.go | 18 +- pkg/satellite/controllers/config.go | 7 + .../device_holds_data_event_internal_test.go | 58 ++++ pkg/satellite/controllers/manager.go | 4 + pkg/satellite/controllers/resource.go | 30 ++- pkg/satellite/reconciler_drbd_test.go | 37 ++- .../reconciler_luks_holds_data_test.go | 81 ++++++ pkg/storage/exec.go | 14 + stand/blockstor-satellite-daemonset.yaml | 3 + 13 files changed, 624 insertions(+), 30 deletions(-) create mode 100644 pkg/luks/luks_holds_data_test.go create mode 100644 pkg/satellite/controllers/device_holds_data_event_internal_test.go create mode 100644 pkg/satellite/reconciler_luks_holds_data_test.go diff --git a/config/rbac/satellite_role.yaml b/config/rbac/satellite_role.yaml index 7d31c1b4..7f8036e0 100644 --- a/config/rbac/satellite_role.yaml +++ b/config/rbac/satellite_role.yaml @@ -103,6 +103,15 @@ rules: verbs: - create - patch +# The same events through events.k8s.io, which the manager's recorder +# writes (the LUKS format refused over data, see resource.go). +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - patch # corev1.Node — read-only access for Bug 283 .res rendering: the # dispatcher's peer-address lookup needs each node's routable # InternalIP to overcome piraeus-operator's pod-CIDR overwrite of diff --git a/pkg/luks/luks.go b/pkg/luks/luks.go index ad37f803..45c5afff 100644 --- a/pkg/luks/luks.go +++ b/pkg/luks/luks.go @@ -24,6 +24,7 @@ package luks import ( "context" + "os" "strings" "github.com/cockroachdb/errors" @@ -33,23 +34,66 @@ import ( // Cryptsetup is a thin wrapper around the `cryptsetup` binary. type Cryptsetup struct { - exec storage.Exec + exec storage.Exec + isBlockDevice func(path string) (bool, error) +} + +// Option configures a Cryptsetup. +type Option func(*Cryptsetup) + +// WithBlockDeviceCheck replaces the check Format makes that a device +// blkid found empty is a block device. Tests whose devices exist only +// in a fake Exec use it. +func WithBlockDeviceCheck(check func(path string) (bool, error)) Option { + return func(c *Cryptsetup) { c.isBlockDevice = check } } // NewCryptsetup constructs a wrapper with the given Exec. -func NewCryptsetup(ex storage.Exec) *Cryptsetup { - return &Cryptsetup{exec: ex} +func NewCryptsetup(ex storage.Exec, opts ...Option) *Cryptsetup { + c := &Cryptsetup{exec: ex, isBlockDevice: statBlockDevice} + for _, opt := range opts { + opt(c) + } + + return c +} + +// statBlockDevice reports whether path names a block device. +func statBlockDevice(path string) (bool, error) { + info, err := os.Stat(path) + if err != nil { + return false, errors.Wrapf(err, "stat %s", path) + } + + mode := info.Mode() + + return mode&os.ModeDevice != 0 && mode&os.ModeCharDevice == 0, nil } // Format runs `cryptsetup luksFormat` against device with the given // keyfile contents. Idempotent: if the device already carries a LUKS // header, we no-op. +// +// A device that is not LUKS but carries any other signature is refused +// with ErrDeviceHoldsData rather than formatted: luksFormat over it +// destroys what is there. See probeSignatures. func (c *Cryptsetup) Format(ctx context.Context, device string, key []byte) error { err := c.runProbe(ctx, "isLuks", device) if err == nil { return nil } + signatures, err := c.probeSignatures(ctx, device) + if err != nil { + return err + } + + if len(signatures) > 0 { + return errors.Wrapf(ErrDeviceHoldsData, + "%s carries %s; refusing luksFormat. If the volume is new and the signature is left over from an earlier one on the same extents, clear it with wipefs", + device, strings.Join(signatures, ", ")) + } + err = c.runWithKey(ctx, key, "luksFormat", "--batch-mode", device, "--key-file", "-") if err != nil { return errors.Wrapf(err, "luksFormat %s", device) @@ -58,6 +102,11 @@ func (c *Cryptsetup) Format(ctx context.Context, device string, key []byte) erro return nil } +// ErrDeviceHoldsData marks a device Format refused because it already +// carries a filesystem, DRBD metadata, a partition table or any other +// signature libblkid recognises. +var ErrDeviceHoldsData = errors.New("luks: device already holds data") + // ErrAlreadyOpen marks the cryptsetup EEXIST class — the dm-crypt // mapper is already active for this device. Returned by Open wrapped // via errors.Wrap so callers can both errors.Is-match the sentinel @@ -149,6 +198,79 @@ func (c *Cryptsetup) Close(ctx context.Context, dmName string) error { return nil } +// blkidNothingFound is the exit status `blkid -p` returns when the +// device carries no signature at all. Every other non-zero status, +// including 8 (several signatures, ambivalent), is either data or a +// probe that could not tell. +// +// blkid also exits 2 when it cannot open the device. It usually says so +// on stderr with blkidErrorMarker, but not for every path: for one that +// does not exist it can exit 2 and print nothing. So exit 2 counts as +// empty only without the marker and for a path that stats as a block +// device; anything else is a probe that could not tell. +const blkidNothingFound = 2 + +// blkidErrorMarker prefixes blkid's own error lines on stderr, which +// storage.Exec folds into the returned error. +const blkidErrorMarker = "blkid: error:" + +// probeSignatures lists the signatures on device with a low-level +// `blkid -p` probe, which reads the device itself rather than the +// cache and sees DRBD internal metadata (TYPE=drbd) as well as +// filesystems. A volume that held data under a stack without LUKS +// shows one or the other. +// +// It fails closed: a probe that errors for any reason other than +// finding nothing returns that error, so Format formats only a device +// it has seen to be empty. +func (c *Cryptsetup) probeSignatures(ctx context.Context, device string) ([]string, error) { + out, err := c.exec.Run(ctx, "blkid", "-p", "-o", "export", device) + if err != nil { + var coded interface{ ExitCode() int } + if errors.As(err, &coded) && coded.ExitCode() == blkidNothingFound && + !strings.Contains(err.Error(), blkidErrorMarker) { + return nil, c.requireBlockDevice(device) + } + + return nil, errors.Wrapf(err, "probe %s for existing data before luksFormat", device) + } + + var signatures []string + + for line := range strings.SplitSeq(string(out), "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if ok && (key == "TYPE" || key == "PTTYPE") { + signatures = append(signatures, key+"="+value) + } + } + + if len(signatures) == 0 { + // blkid reported a find without naming it; still a find. + signatures = append(signatures, "an unnamed signature") + } + + return signatures, nil +} + +// requireBlockDevice stands between blkid's "nothing found" and +// luksFormat: blkid gives the same answer for a path it could not open. +func (c *Cryptsetup) requireBlockDevice(device string) error { + ok, err := c.isBlockDevice(device) + if err != nil { + return errors.Wrapf(err, "probe %s for existing data before luksFormat", device) + } + + if !ok { + return errors.Wrapf(ErrNotABlockDevice, "probe %s for existing data before luksFormat", device) + } + + return nil +} + +// ErrNotABlockDevice marks a device path blkid found empty that is not +// a block device, so its answer says nothing about the volume. +var ErrNotABlockDevice = errors.New("luks: not a block device") + // runProbe runs a status-check subcommand whose exit code is what // matters (cryptsetup isLuks: 0 yes, !=0 no). func (c *Cryptsetup) runProbe(ctx context.Context, args ...string) error { diff --git a/pkg/luks/luks_bug_175_test.go b/pkg/luks/luks_bug_175_test.go index 7aa3f966..e1b894c4 100644 --- a/pkg/luks/luks_bug_175_test.go +++ b/pkg/luks/luks_bug_175_test.go @@ -180,8 +180,10 @@ func TestBug175LUKSFormatPassphraseWithShellMetacharsSafe(t *testing.T) { fx := storage.NewFakeExec() fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Err: errBug175NotLuks}) + fx.Expect("blkid -p -o export /dev/sda", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Format(t.Context(), "/dev/sda", []byte(payloadFormat)) if err != nil { @@ -198,7 +200,7 @@ func TestBug175LUKSFormatPassphraseWithShellMetacharsSafe(t *testing.T) { func TestBug175LUKSOpenPassphraseWithShellMetacharsSafe(t *testing.T) { fx := storage.NewFakeExec() - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Open(t.Context(), "/dev/sda", "pvc-1-crypt", []byte(payloadOpen)) if err != nil { @@ -215,7 +217,7 @@ func TestBug175LUKSOpenPassphraseWithShellMetacharsSafe(t *testing.T) { func TestBug175LUKSResizePassphraseWithShellMetacharsSafe(t *testing.T) { fx := storage.NewFakeExec() - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Resize(t.Context(), "pvc-grow-0-luks", []byte(payloadResize)) if err != nil { @@ -235,8 +237,10 @@ func TestBug175LUKSPassphraseStdinNotArgv(t *testing.T) { fx := storage.NewFakeExec() fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Err: errBug175NotLuks}) + fx.Expect("blkid -p -o export /dev/sda", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) // Mix of three distinct injection shapes — if any single one // leaks via argv we want a localised failure. diff --git a/pkg/luks/luks_holds_data_test.go b/pkg/luks/luks_holds_data_test.go new file mode 100644 index 00000000..e452be94 --- /dev/null +++ b/pkg/luks/luks_holds_data_test.go @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package luks_test + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cozystack/blockstor/pkg/luks" + "github.com/cozystack/blockstor/pkg/storage" +) + +var errProbeIO = errors.New("blkid: /dev/sda: Input/output error") + +const blkidCmd = "blkid -p -o export /dev/sda" + +// anyBlockDevice takes every path for a block device, since the devices +// these tests format exist only in the fake Exec. +func anyBlockDevice() luks.Option { + return luks.WithBlockDeviceCheck(func(string) (bool, error) { return true, nil }) +} + +// formatOutcome is what Format did to a device: its error, and whether +// it ran luksFormat. +type formatOutcome struct { + err error + formatted bool +} + +// formatOver runs Format on a non-LUKS /dev/sda whose blkid probe +// answers with probe. +func formatOver(t *testing.T, probe storage.FakeResponse) formatOutcome { + t.Helper() + + fx := storage.NewFakeExec() + fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Err: errFakeNotLuks}) + fx.Expect(blkidCmd, probe) + + out := formatOutcome{err: luks.NewCryptsetup(fx, anyBlockDevice()).Format(t.Context(), "/dev/sda", []byte("k"))} + + for _, line := range fx.CommandLines() { + if strings.Contains(line, "luksFormat") { + out.formatted = true + } + } + + return out +} + +// A volume that already went to disk without LUKS carries DRBD +// internal metadata, a filesystem, or both. luksFormat over it would +// destroy the data, so Format refuses and names what it found. +func TestFormatRefusesADeviceThatHoldsData(t *testing.T) { + t.Parallel() + + for name, probe := range map[string]struct { + stdout string + want string + }{ + "DRBD internal metadata": {stdout: "DEVNAME=/dev/sda\nTYPE=drbd\n", want: "TYPE=drbd"}, + "filesystem": {stdout: "DEVNAME=/dev/sda\nUUID=1\nTYPE=ext4\nUSAGE=filesystem\n", want: "TYPE=ext4"}, + "partition table": {stdout: "DEVNAME=/dev/sda\nPTTYPE=gpt\n", want: "PTTYPE=gpt"}, + "unnamed signature": {stdout: "", want: "unnamed signature"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := formatOver(t, storage.FakeResponse{Stdout: []byte(probe.stdout)}) + err, formatted := got.err, got.formatted + if !errors.Is(err, luks.ErrDeviceHoldsData) { + t.Fatalf("Format: %v, want ErrDeviceHoldsData", err) + } + + if !strings.Contains(err.Error(), probe.want) { + t.Errorf("error %q does not name %s", err, probe.want) + } + + if formatted { + t.Error("luksFormat ran over a device that holds data") + } + }) + } +} + +// A probe that cannot say the device is empty is no licence to +// format it: several signatures at once (blkid's ambivalent exit 8) or a +// failed read both stop Format. +func TestFormatFailsClosedOnAnInconclusiveProbe(t *testing.T) { + t.Parallel() + + for name, probeErr := range map[string]error{ + "ambivalent": storage.FakeExitError{Code: 8}, + "read error": errProbeIO, + // blkid exits 2 for "nothing found" and for a device it could not + // open; only its stderr tells them apart. + "cannot open": fmt.Errorf("%w: blkid -p -o export /dev/sda: blkid: error: /dev/sda: Permission denied", storage.FakeExitError{Code: 2}), + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := formatOver(t, storage.FakeResponse{Err: probeErr}) + err, formatted := got.err, got.formatted + if err == nil || !errors.Is(err, probeErr) { + t.Fatalf("Format: %v, want the probe's error", err) + } + + if formatted { + t.Error("luksFormat ran after an inconclusive probe") + } + }) + } + + // blkid can exit 2 without a word for a path it could not open, so + // "nothing found" counts only for a path that stats as a block device. + dir := t.TempDir() + regular := filepath.Join(dir, "regular") + + err := os.WriteFile(regular, nil, 0o600) + if err != nil { + t.Fatal(err) + } + + for name, device := range map[string]string{ + "missing path": filepath.Join(dir, "missing"), + "regular file": regular, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + fx := storage.NewFakeExec() + fx.Expect("cryptsetup isLuks "+device, storage.FakeResponse{Err: errFakeNotLuks}) + fx.Expect("blkid -p -o export "+device, storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) + + err := luks.NewCryptsetup(fx).Format(t.Context(), device, []byte("k")) + if err == nil { + t.Fatal("Format succeeded over a path that is not a block device") + } + + for _, line := range fx.CommandLines() { + if strings.Contains(line, "luksFormat") { + t.Error("luksFormat ran over a path that is not a block device") + } + } + }) + } +} + +// An existing LUKS header is the idempotent path: no probe, no format. +func TestFormatOfALUKSDeviceDoesNotProbe(t *testing.T) { + t.Parallel() + + fx := storage.NewFakeExec() + fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{}) + + err := luks.NewCryptsetup(fx, anyBlockDevice()).Format(t.Context(), "/dev/sda", []byte("k")) + if err != nil { + t.Fatalf("Format: %v", err) + } + + for _, line := range fx.CommandLines() { + if strings.HasPrefix(line, "blkid") || strings.Contains(line, "luksFormat") { + t.Errorf("unexpected call on a LUKS device: %s", line) + } + } +} + +// Only "nothing found" lets Format through. +func TestFormatFormatsAnEmptyDevice(t *testing.T) { + t.Parallel() + + got := formatOver(t, storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) + if got.err != nil { + t.Fatalf("Format: %v", got.err) + } + + if !got.formatted { + t.Error("luksFormat did not run on an empty device") + } +} + +// realBlkidStatus runs blkid as a real process that exits with the +// given status, and every other command through the fake: the exit +// status has to survive RealExec's error wrapping to be read. +type realBlkidStatus struct { + *storage.FakeExec + + status string +} + +func (r realBlkidStatus) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + return r.RunWithStdin(ctx, nil, name, args...) +} + +func (r realBlkidStatus) RunWithStdin(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, error) { + var ( + out []byte + err error + ) + + if name == "blkid" { + out, err = storage.RealExec{}.Run(ctx, "sh", "-c", "exit "+r.status) + } else { + out, err = r.FakeExec.RunWithStdin(ctx, stdin, name, args...) + } + + if err != nil { + return out, fmt.Errorf("%s: %w", name, err) + } + + return out, nil +} + +// The empty-device status is read off a real process exit, through +// RealExec's wrapping, not only off the fake. +func TestFormatReadsBlkidsRealExitStatus(t *testing.T) { + t.Parallel() + + for status, wantFormat := range map[string]bool{"2": true, "8": false} { + t.Run("exit "+status, func(t *testing.T) { + t.Parallel() + + fx := storage.NewFakeExec() + fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Err: errFakeNotLuks}) + + err := luks.NewCryptsetup(realBlkidStatus{FakeExec: fx, status: status}, anyBlockDevice()).Format(t.Context(), "/dev/sda", []byte("k")) + if (err == nil) != wantFormat { + t.Fatalf("Format after blkid exit %s: %v", status, err) + } + }) + } +} diff --git a/pkg/luks/luks_test.go b/pkg/luks/luks_test.go index 99f4b486..bb148dfd 100644 --- a/pkg/luks/luks_test.go +++ b/pkg/luks/luks_test.go @@ -36,8 +36,10 @@ func TestFormatRunsLuksFormat(t *testing.T) { fx := storage.NewFakeExec() fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Err: errFakeNotLuks}) + fx.Expect("blkid -p -o export /dev/sda", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Format(t.Context(), "/dev/sda", []byte("supersecret")) if err != nil { @@ -61,7 +63,7 @@ func TestFormatIdempotent(t *testing.T) { fx.Expect("cryptsetup isLuks /dev/sda", storage.FakeResponse{Stdout: []byte("")}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Format(t.Context(), "/dev/sda", []byte("supersecret")) if err != nil { @@ -79,7 +81,7 @@ func TestFormatIdempotent(t *testing.T) { func TestOpenRunsLuksOpen(t *testing.T) { fx := storage.NewFakeExec() - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Open(t.Context(), "/dev/sda", "pvc-1-crypt", []byte("supersecret")) if err != nil { @@ -101,7 +103,7 @@ func TestOpenRunsLuksOpen(t *testing.T) { func TestCloseRunsLuksClose(t *testing.T) { fx := storage.NewFakeExec() - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Close(t.Context(), "pvc-1-crypt") if err != nil { @@ -138,7 +140,7 @@ func TestDevicePath(t *testing.T) { // cryptsetup's audit logs and (worse) into procfs. func TestResizeRunsCryptsetupResize(t *testing.T) { fx := storage.NewFakeExec() - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Resize(t.Context(), "pvc-grow-0-luks", []byte("supersecret")) if err != nil { @@ -187,7 +189,7 @@ func TestOpenErrorWraps(t *testing.T) { fx.Expect("cryptsetup luksOpen /dev/sda pvc-1 --key-file -", storage.FakeResponse{Err: errLuksFailed}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Open(t.Context(), "/dev/sda", "pvc-1", []byte("k")) if err == nil { @@ -205,7 +207,7 @@ func TestCloseErrorWraps(t *testing.T) { fx.Expect("cryptsetup luksClose pvc-1", storage.FakeResponse{Err: errLuksFailed}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Close(t.Context(), "pvc-1") if err == nil { @@ -229,7 +231,7 @@ func TestResizeErrorWraps(t *testing.T) { fx.Expect("cryptsetup resize pvc-1 --key-file -", storage.FakeResponse{Err: errLuksFailed}) - c := luks.NewCryptsetup(fx) + c := luks.NewCryptsetup(fx, anyBlockDevice()) err := c.Resize(t.Context(), "pvc-1", []byte("k")) if err == nil { diff --git a/pkg/satellite/controllers/config.go b/pkg/satellite/controllers/config.go index 6ff27516..b1dd0092 100644 --- a/pkg/satellite/controllers/config.go +++ b/pkg/satellite/controllers/config.go @@ -19,6 +19,7 @@ limitations under the License. package controllers import ( + "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -61,6 +62,12 @@ type Config struct { // nil and the reconciler falls back to the cached client. APIReader client.Reader + // Recorder emits the Warning Event for an apply the satellite refused + // to carry out over data, so it reaches `kubectl describe` rather than + // only the satellite log. Production is wired from the manager in + // NewManager; nil skips the Event. + Recorder events.EventRecorder + // Namespace is where the controller-side Secrets live — in // particular the cluster master passphrase Secret stamped by // `linstor encryption create-passphrase` (Bug 023: the diff --git a/pkg/satellite/controllers/device_holds_data_event_internal_test.go b/pkg/satellite/controllers/device_holds_data_event_internal_test.go new file mode 100644 index 00000000..1324b529 --- /dev/null +++ b/pkg/satellite/controllers/device_holds_data_event_internal_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "strings" + "testing" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" + "github.com/cozystack/blockstor/pkg/luks" + intent "github.com/cozystack/blockstor/pkg/satellite/intent" +) + +// A LUKS format refused over a device that holds data leaves the volume down +// for good, so the refusal reaches the Resource as a Warning Event and not +// only the satellite log. Any other failure stays a log line. +func TestAFormatRefusedOverDataIsAWarningEventOnTheResource(t *testing.T) { + t.Parallel() + + res := &blockstoriov1alpha1.Resource{ObjectMeta: metav1.ObjectMeta{Name: "pvc-1.node-a"}} + + for _, tc := range []struct { + name, message string + event bool + }{ + {name: "holds-data", message: "luks format /dev/vg/pvc-1_00000: carries TYPE=ext4; refusing luksFormat: " + luks.ErrDeviceHoldsData.Error(), event: true}, + {name: "other", message: "drbdadm adjust pvc-1: exit status 10", event: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + recorder := events.NewFakeRecorder(4) + r := &ResourceReconciler{Config: Config{Recorder: recorder}} + + failed := r.recordPerResourceFailures(res, []*intent.ResourceApplyResult{{Name: "pvc-1", Ok: false, Message: tc.message}}, logr.Discard()) + if !failed { + t.Fatal("a failed result was not reported as a failure") + } + + select { + case ev := <-recorder.Events: + if !tc.event { + t.Errorf("unexpected Event %q", ev) + } else if !strings.Contains(ev, "Warning "+deviceHoldsDataReason) { + t.Errorf("Event = %q, want a Warning %s", ev, deviceHoldsDataReason) + } + default: + if tc.event { + t.Error("no Event for a format refused over data") + } + } + }) + } +} diff --git a/pkg/satellite/controllers/manager.go b/pkg/satellite/controllers/manager.go index 34b8ee41..9a1143b8 100644 --- a/pkg/satellite/controllers/manager.go +++ b/pkg/satellite/controllers/manager.go @@ -115,6 +115,10 @@ func NewManager(restCfg *rest.Config, cfg *Config) (manager.Manager, error) { cfg.APIReader = mgr.GetAPIReader() } + if cfg.Recorder == nil { + cfg.Recorder = mgr.GetEventRecorder("blockstor-satellite") + } + // Phase 11.7: thread the observer-trigger channel through // Config so the ObserverRunnable (producer) and the // ResourceReconciler (consumer via WatchesRawSource) share diff --git a/pkg/satellite/controllers/resource.go b/pkg/satellite/controllers/resource.go index 6809d182..d9c2a2ca 100644 --- a/pkg/satellite/controllers/resource.go +++ b/pkg/satellite/controllers/resource.go @@ -49,6 +49,7 @@ import ( apiv1 "github.com/cozystack/blockstor/pkg/api/v1" "github.com/cozystack/blockstor/pkg/dispatcher" "github.com/cozystack/blockstor/pkg/effectiveprops" + "github.com/cozystack/blockstor/pkg/luks" intent "github.com/cozystack/blockstor/pkg/satellite/intent" ) @@ -508,7 +509,7 @@ func (r *ResourceReconciler) runApply(ctx context.Context, res *blockstoriov1alp return ctrl.Result{}, errors.Wrap(err, "satellite Apply") } - anyFailed := r.recordPerResourceFailures(results, logger) + anyFailed := r.recordPerResourceFailures(res, results, logger) r.stampPostApply(ctx, res, &rd, results, anyFailed, logger) @@ -590,20 +591,37 @@ func (r *ResourceReconciler) prepareDRBDApply( // recordPerResourceFailures logs each per-resource Apply failure and // returns whether any of the results came back not-Ok. Extracted from // runApply so the orchestration stays under the funlen budget. -func (r *ResourceReconciler) recordPerResourceFailures(results []*intent.ResourceApplyResult, logger logr.Logger) bool { +// +// A LUKS layer that refused to format a device holding data also gets a +// Warning Event on the Resource: the volume never comes up, and an Info +// line in this log is not where an operator looks for why. +func (r *ResourceReconciler) recordPerResourceFailures( + res *blockstoriov1alpha1.Resource, results []*intent.ResourceApplyResult, logger logr.Logger, +) bool { var anyFailed bool - for _, ar := range results { - if !ar.GetOk() { - anyFailed = true + for _, result := range results { + if result.GetOk() { + continue + } + + anyFailed = true + + logger.Info("Apply per-resource failure", "name", result.GetName(), "message", result.GetMessage()) - logger.Info("Apply per-resource failure", "name", ar.GetName(), "message", ar.GetMessage()) + if r.Config.Recorder != nil && strings.Contains(result.GetMessage(), luks.ErrDeviceHoldsData.Error()) { + r.Config.Recorder.Eventf(res, nil, corev1.EventTypeWarning, deviceHoldsDataReason, "Apply", + "the LUKS layer refused to format a device that already holds data: %s", result.GetMessage()) } } return anyFailed } +// deviceHoldsDataReason is the Event reason for a LUKS format refused over +// a device that carries a signature. +const deviceHoldsDataReason = "LUKSDeviceHoldsData" + // stampPostApply runs the four best-effort post-apply stamps: // Status.Volumes (per-volume DevicePath), the stale-Status.Volumes // purge (Bug 355 followup), the Bug-107 volume-numbers annotation, diff --git a/pkg/satellite/reconciler_drbd_test.go b/pkg/satellite/reconciler_drbd_test.go index 445863cb..26a50c38 100644 --- a/pkg/satellite/reconciler_drbd_test.go +++ b/pkg/satellite/reconciler_drbd_test.go @@ -117,6 +117,13 @@ var ( // expects the day0 skip / winner set-gi seed must mark its fixture // SkipInitialSync=true (the controller decided this is a genuinely-fresh // RD), exactly as the real controller stamps it on the initial set. + +// anyBlockDevice takes every path for a block device, since the LUKS +// devices these tests format exist only in the fake Exec. +func anyBlockDevice() luks.Option { + return luks.WithBlockDeviceCheck(func(string) (bool, error) { return true, nil }) +} + func skipInitTrue() *bool { v := true; return &v } func skipInitFalse() *bool { v := false; return &v } @@ -907,6 +914,8 @@ func TestApplyLayersLUKS(t *testing.T) { // cryptsetup isLuks fails on a fresh device → format runs. fx.Expect("cryptsetup isLuks /dev/vg/pvc-luks_00000", storage.FakeResponse{Err: errNotALUKSDevice}) + fx.Expect("blkid -p -o export /dev/vg/pvc-luks_00000", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) thin := lvm.NewThin(lvm.ThinConfig{VolumeGroup: "vg", ThinPool: "tp"}, fx) rec := satellite.NewReconciler(satellite.ReconcilerConfig{ @@ -914,7 +923,7 @@ func TestApplyLayersLUKS(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) _, err := rec.Apply(t.Context(), []*intent.DesiredResource{ @@ -966,7 +975,7 @@ func TestApplyLUKSFailsWithoutPassphrase(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) results, err := rec.Apply(t.Context(), []*intent.DesiredResource{ @@ -1011,6 +1020,8 @@ func TestApplyLUKSStorageNeverDRBD(t *testing.T) { // First reconcile: not yet a LUKS device → luksFormat will run. fx.Expect("cryptsetup isLuks /dev/vg/pvc-luks-only_00000", storage.FakeResponse{Err: errNotALUKSDevice}) + fx.Expect("blkid -p -o export /dev/vg/pvc-luks-only_00000", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) thin := lvm.NewThin(lvm.ThinConfig{VolumeGroup: "vg", ThinPool: "tp"}, fx) rec := satellite.NewReconciler(satellite.ReconcilerConfig{ @@ -1018,7 +1029,7 @@ func TestApplyLUKSStorageNeverDRBD(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) dr := []*intent.DesiredResource{ @@ -1100,6 +1111,8 @@ func TestApplyDRBDLUKSStorageStack(t *testing.T) { storage.FakeResponse{Stdout: []byte("/dev/vg/pvc-stack_00000|1048576\n")}) fx.Expect("cryptsetup isLuks /dev/vg/pvc-stack_00000", storage.FakeResponse{Err: errNotALUKSDevice}) + fx.Expect("blkid -p -o export /dev/vg/pvc-stack_00000", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) thin := lvm.NewThin(lvm.ThinConfig{VolumeGroup: "vg", ThinPool: "tp"}, fx) rec := satellite.NewReconciler(satellite.ReconcilerConfig{ @@ -1107,7 +1120,7 @@ func TestApplyDRBDLUKSStorageStack(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) _, err := rec.Apply(t.Context(), []*intent.DesiredResource{ @@ -1609,7 +1622,7 @@ func TestApplyLUKSResizeChainsThroughMapper(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) _, err := rec.Apply(t.Context(), []*intent.DesiredResource{ @@ -1706,7 +1719,7 @@ func TestApplyLUKSConvergeResizeAfterCrashRecovery(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), Exec: fx, }) @@ -1803,7 +1816,7 @@ func TestApplyLUKSConvergeNoOpWhenMapperMatches(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), Exec: fx, }) @@ -2329,7 +2342,7 @@ func TestDeleteResourceClosesLUKSMapper(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) resp, err := rec.DeleteResource(t.Context(), &intent.DeleteResourceRequest{ @@ -2599,6 +2612,8 @@ func TestApplyLUKSFormatErrorWraps(t *testing.T) { storage.FakeResponse{Stdout: []byte("/dev/vg/pvc-luks-format_00000|1048576\n")}) fx.Expect("cryptsetup isLuks /dev/vg/pvc-luks-format_00000", storage.FakeResponse{Err: errNotALUKSDevice}) + fx.Expect("blkid -p -o export /dev/vg/pvc-luks-format_00000", + storage.FakeResponse{Err: storage.FakeExitError{Code: 2}}) // luksFormat fails: simulate the device being busy. Post-Issue // 175 the cryptsetup invocation is direct, so the Expect key is // the cryptsetup argv verbatim with no shell pipeline wrapping. @@ -2611,7 +2626,7 @@ func TestApplyLUKSFormatErrorWraps(t *testing.T) { Adm: drbd.NewAdm(fx), StateDir: dir, NodeName: "n1", - Cryptsetup: luks.NewCryptsetup(fx), + Cryptsetup: luks.NewCryptsetup(fx, anyBlockDevice()), }) results, err := rec.Apply(t.Context(), []*intent.DesiredResource{ @@ -2637,6 +2652,10 @@ func TestApplyLUKSFormatErrorWraps(t *testing.T) { t.Errorf("Ok: got true, want false on luksFormat failure") } + if !strings.Contains(results[0].GetMessage(), errLUKSFormatBusy.Error()) { + t.Errorf("message %q is not the luksFormat failure", results[0].GetMessage()) + } + if !strings.Contains(results[0].GetMessage(), "luks format") { t.Errorf("message: got %q, want substring \"luks format\"", results[0].GetMessage()) } diff --git a/pkg/satellite/reconciler_luks_holds_data_test.go b/pkg/satellite/reconciler_luks_holds_data_test.go new file mode 100644 index 00000000..6d8e45f4 --- /dev/null +++ b/pkg/satellite/reconciler_luks_holds_data_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package satellite_test + +import ( + "strings" + "testing" + + "github.com/cozystack/blockstor/pkg/drbd" + "github.com/cozystack/blockstor/pkg/luks" + "github.com/cozystack/blockstor/pkg/satellite" + intent "github.com/cozystack/blockstor/pkg/satellite/intent" + "github.com/cozystack/blockstor/pkg/storage" + "github.com/cozystack/blockstor/pkg/storage/lvm" +) + +// A replica that already went to disk under [DRBD,STORAGE] and is now +// asked for LUKS keeps its data: the apply fails on that volume with +// the reason in the message, and luksFormat never runs. +func TestApplyLUKSOverAPlaintextVolumeIsRefused(t *testing.T) { + t.Parallel() + + const dev = "/dev/vg/pvc-plain_00000" + + fx := storage.NewFakeExec() + fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg/pvc-plain_00000", + storage.FakeResponse{Stdout: []byte("pvc-plain_00000\n")}) + fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings --separator | -o lv_path,lv_size --units k --nosuffix vg/pvc-plain_00000", + storage.FakeResponse{Stdout: []byte(dev + "|1048576\n")}) + fx.Expect("cryptsetup isLuks "+dev, storage.FakeResponse{Err: errNotALUKSDevice}) + fx.Expect("blkid -p -o export "+dev, storage.FakeResponse{Stdout: []byte("DEVNAME=" + dev + "\nTYPE=drbd\n")}) + + rec := satellite.NewReconciler(satellite.ReconcilerConfig{ + Providers: map[string]storage.Provider{"thin1": lvm.NewThin(lvm.ThinConfig{VolumeGroup: "vg", ThinPool: "tp"}, fx)}, + Adm: drbd.NewAdm(fx), + StateDir: t.TempDir(), + NodeName: "n1", + Cryptsetup: luks.NewCryptsetup(fx), + }) + + results, err := rec.Apply(t.Context(), []*intent.DesiredResource{{ + Name: "pvc-plain", + NodeName: "n1", + Volumes: []*intent.DesiredVolume{{VolumeNumber: 0, SizeKib: 1024 * 1024, StoragePool: "thin1"}}, + LayerStack: []string{"LUKS", "STORAGE"}, + Props: map[string]string{"LuksPassphrase": "topsecret"}, + }}) + if err != nil { + t.Fatalf("Apply: %v", err) + } + + if len(results) != 1 || results[0].GetOk() { + t.Fatalf("results %v, want one failed apply", results) + } + + if !strings.Contains(results[0].GetMessage(), "already holds data") || !strings.Contains(results[0].GetMessage(), "TYPE=drbd") { + t.Errorf("message %q does not say the device holds DRBD data", results[0].GetMessage()) + } + + for _, line := range fx.CommandLines() { + if strings.Contains(line, "luksFormat") { + t.Fatalf("luksFormat ran over a plaintext volume: %s", line) + } + } +} diff --git a/pkg/storage/exec.go b/pkg/storage/exec.go index e5c13799..1a7ee1ec 100644 --- a/pkg/storage/exec.go +++ b/pkg/storage/exec.go @@ -24,6 +24,7 @@ import ( "io" "os" "os/exec" + "strconv" "strings" "sync" @@ -148,6 +149,19 @@ type FakeResponse struct { Err error } +// FakeExitError is a canned non-zero exit for FakeResponse.Err. It +// carries the status the way *exec.ExitError does, so code that reads +// a command's exit status (blkid's "nothing found" is 2) can be driven +// without running the command. +type FakeExitError struct { + Code int +} + +func (e FakeExitError) Error() string { return "exit status " + strconv.Itoa(e.Code) } + +// ExitCode returns the canned status, as (*exec.ExitError).ExitCode does. +func (e FakeExitError) ExitCode() int { return e.Code } + // NewFakeExec returns a FakeExec ready for use. func NewFakeExec() *FakeExec { return &FakeExec{Responses: map[string]FakeResponse{}} diff --git a/stand/blockstor-satellite-daemonset.yaml b/stand/blockstor-satellite-daemonset.yaml index 33a91a6c..07d7202e 100644 --- a/stand/blockstor-satellite-daemonset.yaml +++ b/stand/blockstor-satellite-daemonset.yaml @@ -61,6 +61,9 @@ rules: - apiGroups: [""] resources: [events] verbs: [create, patch] + - apiGroups: [events.k8s.io] + resources: [events] + verbs: [create, patch] - apiGroups: [""] resources: [secrets] verbs: [get, list, watch] From a99361262b872da81efd370b3a272a83502bdf47 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 13 Oct 2026 00:03:50 +0200 Subject: [PATCH 04/22] fix(controller): count an initialized definition as holding data The layer stack pin looked only at what replicas had reported. A definition can hold data before any of them has: linstor-migrate adopts volumes that are already full and latches Initialized on them, and the latch is set once the controller has seen a real write. Either way the storage was built under the stack the definition carried, so the pin keeps it. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- .../resourcedefinition_layer_stack.go | 32 +++++++++++++++---- .../resourcedefinition_layer_stack_test.go | 19 +++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index d2069c0c..786040f2 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -82,16 +82,11 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl return nil } - holdsData, err := r.rdHoldsData(ctx, rd.Name) + stack, err := r.stackToPin(ctx, rd, groupStack) if err != nil { return err } - stack := groupStack - if holdsData { - stack = apiv1.DefaultLayerStack() - } - base := rd.DeepCopy() rd.Spec.LayerStack = slices.Clone(stack) @@ -121,10 +116,33 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl return nil } +// stackToPin is the group's stack for an RD with nothing on disk, and +// the default stack, the one an empty stack was built as, for an RD +// that already holds data. +func (r *ResourceDefinitionReconciler) stackToPin(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition, groupStack []string) ([]string, error) { + if rdInitialized(rd) { + return apiv1.DefaultLayerStack(), nil + } + + holdsData, err := r.rdHoldsData(ctx, rd.Name) + if err != nil { + return nil, err + } + + if holdsData { + return apiv1.DefaultLayerStack(), nil + } + + return groupStack, nil +} + // rdHoldsData reports whether any replica of the RD has gone to disk: // DRBD metadata created, or a volume the satellite reported a device // for. Either means the storage under it was built with the stack the -// RD carried then. +// RD carried then. The caller checks the Initialized latch first, which +// covers the case no replica has reported yet: an RD adopted by +// linstor-migrate, or one whose data was written before its replicas' +// status was. func (r *ResourceDefinitionReconciler) rdHoldsData(ctx context.Context, rdName string) (bool, error) { var list blockstoriov1alpha1.ResourceList diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go index 7aeba1d6..11fa54ea 100644 --- a/internal/controller/resourcedefinition_layer_stack_test.go +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -374,3 +374,22 @@ func TestPinLayerStackDoesNotOverwriteARacingWriter(t *testing.T) { t.Fatalf("stack %v, want the racing writer's", got.rd.Spec.LayerStack) } } + +// The Initialized latch is data too, before any replica has reported: +// an RD adopted by linstor-migrate carries it from the start. +func TestPinLayerStackTreatsTheInitializedLatchAsData(t *testing.T) { + t.Parallel() + + initialized := true + rd := pinRD("rg-pin") + rd.Spec.Initialized = &initialized + + got := reconcilePin(t, pinClient(t, pinGroup(luksStack...), rd, pinReplica("pvc-pin", "n1"))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if !slices.Equal(got.rd.Spec.LayerStack, apiv1.DefaultLayerStack()) { + t.Fatalf("stack %v, want the applied default", got.rd.Spec.LayerStack) + } +} From 802dfb6631f4086d3fef76082cd0c301cca6b9c9 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 13 Oct 2026 00:08:06 +0200 Subject: [PATCH 05/22] fix(linstor-migrate): read an empty definition stack off its replicas The converter copied the resource definition's layer_stack column as is. LINSTOR resolves the stack per resource and often leaves that column empty, so the stack those volumes were built with lives only in LAYER_RESOURCE_IDS. A converted definition with an empty stack then took whatever blockstor resolved for it, not what is on disk: a LUKS volume lost its LUKS layer, and a definition in a group with another stack was read through that group's. An empty stack is now read off the replicas' layer chains. Replicas built in different ways leave it unset and are reported, as is a replica built otherwise than the stack its definition names, since the satellite would not bring it up as it is. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/linstormigrate/convert.go | 93 +++++++++++++- pkg/linstormigrate/layer_stack_test.go | 171 +++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 4 deletions(-) create mode 100644 pkg/linstormigrate/layer_stack_test.go diff --git a/pkg/linstormigrate/convert.go b/pkg/linstormigrate/convert.go index a815539b..d3558f44 100644 --- a/pkg/linstormigrate/convert.go +++ b/pkg/linstormigrate/convert.go @@ -128,6 +128,10 @@ type converter struct { drbdNode map[replicaKey]*int32 // node-id by (node, rd) storVol map[volumeReplicaKey]LayerStorageVolumeRow luksVol map[volumeReplicaKey]LayerLuksVolumeRow + // diskStacks is the layer stack each live replica of an RD was + // actually built with, read off its LAYER_RESOURCE_IDS chain: by + // resource_name, then by the joined stack, to the stack itself. + diskStacks map[string]map[string][]string // referential-integrity sets, populated as each kind converts so // later kinds can drop rows that would dangle (a replica of a @@ -209,6 +213,7 @@ func ConvertWithOptions(dump *Dump, opts Options) (*Result, error) { drbdNode: map[replicaKey]*int32{}, storVol: map[volumeReplicaKey]LayerStorageVolumeRow{}, luksVol: map[volumeReplicaKey]LayerLuksVolumeRow{}, + diskStacks: map[string]map[string][]string{}, convertedRD: map[string]bool{}, convertedNode: map[string]bool{}, convertedRG: map[string]bool{}, @@ -366,6 +371,8 @@ func (c *converter) buildLayerInstanceIndexes() { return lri, true } + c.indexDiskStacks() + for i := range c.dump.LayerDrbdResources { row := &c.dump.LayerDrbdResources[i] if lri, ok := baseLayer(row.LayerResourceID); ok { @@ -388,6 +395,83 @@ func (c *converter) buildLayerInstanceIndexes() { } } +// indexDiskStacks reads the layer stack every live replica was built +// with off LAYER_RESOURCE_IDS: the replica's root layer (no parent), +// then each base-suffix child down to STORAGE. The RD row's own +// layer_stack is only what was asked for at creation, and is often +// empty, LINSTOR having resolved the stack per resource; this chain is +// what is on disk. +func (c *converter) indexDiskStacks() { + rows := c.dump.LayerResourceIDs + children := map[int32]LayerResourceIDRow{} + + for i := range rows { + if rows[i].LayerResourceParentID != nil && rows[i].LayerResourceSuffix == "" { + children[*rows[i].LayerResourceParentID] = rows[i] + } + } + + for i := range rows { + row := rows[i] + if row.LayerResourceParentID != nil || row.SnapshotName != "" || row.LayerResourceSuffix != "" { + continue + } + + var stack []string + + // Bounded by the row count, so a corrupt chain that loops ends. + for cur, ok := row, true; ok && len(stack) <= len(rows); cur, ok = children[cur.LayerResourceID] { + stack = append(stack, cur.LayerResourceKind) + } + + if c.diskStacks[row.ResourceName] == nil { + c.diskStacks[row.ResourceName] = map[string][]string{} + } + + c.diskStacks[row.ResourceName][strings.Join(stack, ",")] = stack + } +} + +// rdLayerStack is the layer stack a converted RD carries. An RD row +// that names one keeps it; an empty one takes the stack its replicas +// were built with, so the satellite adopts each volume as it is rather +// than as the default, and the group's stack, which blockstor would +// otherwise pin, is never laid over data built without it. Replicas +// that disagree, or a stack the replicas were not built with, are +// reported for the operator to settle before adopting. +func (c *converter) rdLayerStack(row *ResourceDefinitionRow, dsp string) []string { + stack := c.parseList(row.LayerStack, "resource definition "+dsp+" layer_stack") + onDisk := c.diskStacks[row.ResourceName] + + keys := make([]string, 0, len(onDisk)) + for key := range onDisk { + keys = append(keys, key) + } + + slices.Sort(keys) + + if len(stack) > 0 { + for _, key := range keys { + if key != strings.Join(stack, ",") { + c.warnf("resource definition %s: layer stack %s, but a replica was built as %s — that replica will not come up as it is", dsp, strings.Join(stack, ","), key) + } + } + + return stack + } + + switch len(keys) { + case 0: + return nil + case 1: + return slices.Clone(onDisk[keys[0]]) + default: + c.warnf("resource definition %s: no layer stack of its own, and its replicas were built as %s — left unset; set spec.layerStack before adopting", dsp, strings.Join(keys, " and ")) + + return nil + } +} + func (c *converter) convertNodes() []crdv1alpha1.Node { nodes := make([]crdv1alpha1.Node, 0, len(c.dump.Nodes)) @@ -937,7 +1021,7 @@ func (c *converter) convertResourceDefinitions() []crdv1alpha1.ResourceDefinitio Props: residual, DRBDOptions: typed, ExtraProps: extra, - LayerStack: c.parseList(row.LayerStack, "resource definition "+dsp+" layer_stack"), + LayerStack: c.rdLayerStack(row, dsp), // The volumes already hold committed data in the source // cluster: latch Initialized so any replica added AFTER // the migration must SyncTarget the real data instead of @@ -1191,9 +1275,10 @@ func (c *converter) attachRDLayerFields(def *crdv1alpha1.ResourceDefinition, row // LAYER_LUKS_VOLUMES carries the volume passphrase encrypted // with the LINSTOR master key; decrypting it needs the // operator's master passphrase and LINSTOR's KDF, which this - // converter does not implement yet. The layer stack is - // preserved, but the volume cannot be opened until the - // passphrase is provisioned into blockstor by hand. + // converter does not implement yet. The layer stack carries + // LUKS (rdLayerStack reads it off the replicas when the RD + // row leaves it empty), but the volume cannot be opened until + // the passphrase is provisioned into blockstor by hand. c.warnf("resource definition %s: LUKS passphrase NOT migrated (encrypted with the LINSTOR master key) — provision spec.encryption manually before adopting", dsp) } } diff --git a/pkg/linstormigrate/layer_stack_test.go b/pkg/linstormigrate/layer_stack_test.go new file mode 100644 index 00000000..c4cc5dba --- /dev/null +++ b/pkg/linstormigrate/layer_stack_test.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package linstormigrate + +import ( + "slices" + "testing" +) + +// layerChain is one replica's LAYER_RESOURCE_IDS chain on node, ids +// from first, root first. +func layerChain(first int32, node, rd string, kinds ...string) []LayerResourceIDRow { + rows := make([]LayerResourceIDRow, 0, len(kinds)) + + for i, kind := range kinds { + row := LayerResourceIDRow{LayerResourceID: first + int32(i), LayerResourceKind: kind, NodeName: node, ResourceName: rd} + if i > 0 { + row.LayerResourceParentID = ptr(first + int32(i) - 1) + } + + rows = append(rows, row) + } + + return rows +} + +// stackDump is a one-RD dump on two nodes whose RD row asks for +// rdStack and whose replicas are built as chains. +func stackDump(rdStack string, chains ...[]LayerResourceIDRow) *Dump { + dump := &Dump{ + Nodes: []NodeRow{ + {NodeName: "NODE-A", NodeDspName: "node-a", NodeType: 2, UUID: "n-a"}, + {NodeName: "NODE-B", NodeDspName: "node-b", NodeType: 2, UUID: "n-b"}, + }, + ResourceGroups: []ResourceGroupRow{ + {ResourceGroupName: "RG", ResourceGroupDspName: "rg", LayerStack: `["STORAGE"]`, UUID: "rg"}, + }, + ResourceDefinitions: []ResourceDefinitionRow{ + {ResourceName: "PVC-S", ResourceDspName: "pvc-s", ResourceGroupName: "RG", LayerStack: rdStack, UUID: "rd-s"}, + }, + VolumeDefinitions: []VolumeDefinitionRow{ + {ResourceName: "PVC-S", VlmNr: 0, VlmSize: 1048576, UUID: "vd-s"}, + }, + } + + for _, chain := range chains { + dump.LayerResourceIDs = append(dump.LayerResourceIDs, chain...) + } + + return dump +} + +func convertedStack(t *testing.T, dump *Dump) ([]string, *Result) { + t.Helper() + + res, err := Convert(dump) + if err != nil { + t.Fatalf("Convert: %v", err) + } + + for _, rd := range res.ResourceDefinitions { + if rd.Name == "pvc-s" { + return rd.Spec.LayerStack, res + } + } + + t.Fatalf("pvc-s did not convert; warnings: %v", res.Warnings) + + return nil, nil +} + +// LINSTOR resolves the stack per resource and often leaves the RD row +// empty, in a group whose own stack is something else (seen on a real +// cluster: a [STORAGE] group holding DRBD volumes). The converted RD +// carries what is on disk, so blockstor never pins the group's stack +// over it. +func TestAnEmptyRDStackIsReadOffItsReplicas(t *testing.T) { + t.Parallel() + + for name, kinds := range map[string][]string{ + "DRBD": {"DRBD", "STORAGE"}, + "DRBD over LUKS": {"DRBD", "LUKS", "STORAGE"}, + "LUKS only": {"LUKS", "STORAGE"}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got, res := convertedStack(t, stackDump("[]", + layerChain(1, "NODE-A", "PVC-S", kinds...), layerChain(10, "NODE-B", "PVC-S", kinds...))) + if !slices.Equal(got, kinds) { + t.Fatalf("stack %v, want %v; warnings %v", got, kinds, res.Warnings) + } + }) + } +} + +// Snapshot instances and suffixed children (DRBD external metadata) +// are not the replica's data path and do not shape its stack. +func TestSnapshotAndSuffixedLayersDoNotShapeTheStack(t *testing.T) { + t.Parallel() + + meta := LayerResourceIDRow{LayerResourceID: 20, LayerResourceKind: "STORAGE", LayerResourceParentID: ptr(int32(1)), LayerResourceSuffix: ".meta", NodeName: "NODE-A", ResourceName: "PVC-S"} + snap := LayerResourceIDRow{LayerResourceID: 21, LayerResourceKind: "STORAGE", NodeName: "NODE-A", ResourceName: "PVC-S", SnapshotName: "SNAP"} + + got, res := convertedStack(t, stackDump("[]", layerChain(1, "NODE-A", "PVC-S", "DRBD", "LUKS", "STORAGE"), []LayerResourceIDRow{meta, snap})) + if !slices.Equal(got, []string{"DRBD", "LUKS", "STORAGE"}) { + t.Fatalf("stack %v; warnings %v", got, res.Warnings) + } +} + +// Replicas built two ways leave nothing to read: the stack stays unset +// and the operator is told which ways. +func TestReplicasThatDisagreeLeaveTheStackUnset(t *testing.T) { + t.Parallel() + + got, res := convertedStack(t, stackDump("[]", + layerChain(1, "NODE-A", "PVC-S", "DRBD", "LUKS", "STORAGE"), layerChain(10, "NODE-B", "PVC-S", "DRBD", "STORAGE"))) + if len(got) != 0 { + t.Fatalf("stack %v, want unset", got) + } + + if !hasWarning(res, "built as DRBD,LUKS,STORAGE and DRBD,STORAGE") { + t.Errorf("disagreement not reported; warnings %v", res.Warnings) + } +} + +// A stack the RD row names is kept, and a replica built otherwise is +// reported: the satellite would try to rebuild it. +func TestAReplicaBuiltOtherwiseThanItsRDIsReported(t *testing.T) { + t.Parallel() + + got, res := convertedStack(t, stackDump(`["DRBD","LUKS","STORAGE"]`, + layerChain(1, "NODE-A", "PVC-S", "DRBD", "LUKS", "STORAGE"), layerChain(10, "NODE-B", "PVC-S", "DRBD", "STORAGE"))) + if !slices.Equal(got, []string{"DRBD", "LUKS", "STORAGE"}) { + t.Fatalf("stack %v, want the RD row's", got) + } + + if !hasWarning(res, "a replica was built as DRBD,STORAGE") { + t.Errorf("mismatch not reported; warnings %v", res.Warnings) + } + + if hasWarning(res, "a replica was built as DRBD,LUKS,STORAGE") { + t.Errorf("a matching replica was reported; warnings %v", res.Warnings) + } +} + +// No replica, nothing to read: the RD stays as the row has it. +func TestAnEmptyRDStackWithoutReplicasStaysEmpty(t *testing.T) { + t.Parallel() + + got, _ := convertedStack(t, stackDump("[]")) + if len(got) != 0 { + t.Fatalf("stack %v, want unset", got) + } +} From 586510b581ae6b2944ed9d3f95c3327dedae9196 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 13 Oct 2026 00:09:02 +0200 Subject: [PATCH 06/22] docs(layer-stack): say where a definition's stack is decided The page said an RD with an empty stack inherits its group's at read time, which is what put plaintext on disk under encrypted definitions. It now says the stack is fixed once on the definition, by the REST create path or by the controller's pin, what happens to a definition that already holds data, what the operator sees, and how to find such definitions. The migration guide and the REST create path's comment say the same. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 22 ++++++++++++++++++---- docs/linstor-migration.md | 2 +- pkg/rest/resource_definitions.go | 17 +++++++++-------- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index d1fc9ce6..8941df8e 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -27,10 +27,24 @@ where it's the network leg). | `["DRBD","LUKS","STORAGE"]` | Encrypted at-rest + replicated. Per-volume cipher. | | `["STORAGE"]` | Single-replica local mode. Ephemeral cache, scratch, ZFS dataset. | -The default is `["DRBD","STORAGE"]` — when `RD.Spec.LayerStack` is -empty the controller inherits from the parent -`ResourceGroup.Spec.SelectFilter.LayerStack`, and when both are empty -the dispatcher falls through to the satellite's default-DRBD path. +The default is `["DRBD","STORAGE"]`, used when neither the RD nor its resource group sets a stack. + +An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The REST create and `rg spawn` paths copy the group's stack onto the RD when it is created. An RD that ends up with an empty stack in a group that sets one (created with `kubectl apply` or GitOps, or moved by `rd modify --resource-group`) is pinned by the controller, and the satellite builds nothing for it until then: + +- An RD that holds no data yet gets the group's stack. +- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. The group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. + +Changing a group's stack later never reshapes RDs that already carry one, as in upstream LINSTOR, where the stack is decided when a resource is created. + +As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. On a fresh thick LV that reuses extents of an earlier volume the signature can be stale; clear it with `wipefs -a ` once you have checked it holds nothing you need. + +Find the RDs that kept the default stack: + +```bash +kubectl get resourcedefinitions.blockstor.cozystack.io \ + -o custom-columns='NAME:.metadata.name,GROUP_STACK:.metadata.annotations.blockstor\.io/unapplied-group-layer-stack' \ + | awk '$2 != ""' +``` ## Setting the stack diff --git a/docs/linstor-migration.md b/docs/linstor-migration.md index 3d902cc6..d482de36 100644 --- a/docs/linstor-migration.md +++ b/docs/linstor-migration.md @@ -10,7 +10,7 @@ Migrated: nodes, storage pools, resource groups, resource definitions (+ volume **NOT migrated — read the report on stderr for the exact list per cluster:** -- **LUKS passphrases.** LUKS-encrypted volumes convert with their layer stack intact, but the per-volume passphrase (encrypted with the LINSTOR master key) is not decrypted. You must provision `spec.encryption` for those RDs by hand before the encrypted volumes can be opened. Each affected RD is reported. +- **LUKS passphrases.** LUKS-encrypted volumes convert with their layer stack intact (an RD whose `layer_stack` column is empty takes the stack its replicas were built with, read from `LAYER_RESOURCE_IDS`), but the per-volume passphrase (encrypted with the LINSTOR master key) is not decrypted. You must provision `spec.encryption` for those RDs by hand before the encrypted volumes can be opened. Each affected RD is reported. - **Backup-shipping remotes**, unknown/runtime flag bits, and DELETE'd / FAILED_DEPLOYMENT rows are skipped and reported. None affect live data availability. - **DRBD TCP ports** are absent from most LINSTOR 1.33 dumps — see "Capture live DRBD ports" below. diff --git a/pkg/rest/resource_definitions.go b/pkg/rest/resource_definitions.go index 8114d3f1..1808644b 100644 --- a/pkg/rest/resource_definitions.go +++ b/pkg/rest/resource_definitions.go @@ -648,17 +648,18 @@ func (s *Server) ensureDefaultRGAssignment(ctx context.Context, rd *apiv1.Resour // inheritLayerStackFromRG copies the parent RG's SelectFilter.LayerStack // onto the RD when the RD itself didn't carry one. Caller-supplied -// rd.LayerStack wins (explicit > inherited), matching the same precedence -// rule v1.ResolveLayerStack applies at the dispatch read-side. The result -// is stamped onto the stored RD so a later `linstor rg modify` that flips -// the parent's LayerStack does NOT retroactively re-layer existing RDs — -// "sticky at create time", just like upstream LINSTOR's behaviour. +// rd.LayerStack wins (explicit > inherited). The result is stamped onto +// the stored RD so a later `linstor rg modify` that flips the parent's +// LayerStack does NOT retroactively re-layer existing RDs — "sticky at +// create time", just like upstream LINSTOR's behaviour. The satellite +// reads only the stamped value and never resolves through the group. // // NotFound on the parent RG is swallowed: the lazily-created DfltRscGrp // in ensureDefaultRGAssignment may race with this lookup on a sibling -// apiserver replica, and the default RG carries no SelectFilter anyway, -// so silently falling through leaves rd.LayerStack empty (= the legacy -// DRBD+STORAGE default, applied downstream). +// apiserver replica, and the default RG carries no SelectFilter anyway. +// An RD left empty here, or created outside REST, is pinned by the +// controller (ResourceDefinitionReconciler.pinLayerStack) before the +// satellite builds anything for it. func (s *Server) inheritLayerStackFromRG(ctx context.Context, rd *apiv1.ResourceDefinition) error { if len(rd.LayerStack) > 0 { return nil From b958c6e013f67c7f776faabc406ac0cbb056b3d5 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:27:19 +0500 Subject: [PATCH 07/22] fix(clone,restore): take the source's decided stack, never a guess A clone or restore wrote the source's stack onto the new definition as stored, reading an empty one as the default. Under a group that sets a stack, an empty one is not decided yet: the controller pins it to the group's stack or to the default depending on whether data is already there, and the satellite builds nothing until it does. The copy took the default regardless and kept it for good, so a shell of an empty definition in a LUKS group stayed plaintext with no data forcing it. The stack is now resolved through the group. An undecided source, leftover or prepared target is answered 503 (the CLI exits with an error) before anything is written, and a group that cannot be read is an error rather than the default. A volume-less clone copies no bytes, so it takes only a stack the source chose and is otherwise left for its own group to decide. A copy short of what its group asks for carries the unapplied-group-layer-stack annotation the controller puts on a definition pinned over data. The restore marker goes onto a prepared target only if, in the patch that writes it, the target's stack still has the source's layers, so a stack rewritten after the target was judged is refused rather than restored into. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- internal/cli/snapshot.go | 34 +- .../cli/snapshot_restore_prepared_test.go | 58 ++- pkg/rest/nodes.go | 12 + pkg/rest/rd_clone.go | 131 +++++-- pkg/rest/rd_clone_validation_test.go | 38 ++ pkg/rest/restore_effective_layers_test.go | 359 +++++++++++++++--- pkg/rest/rg_deleted_race_test.go | 34 +- pkg/rest/snapshot_restore.go | 99 +++-- pkg/store/effective_layer_stack_test.go | 234 ++++++++++++ pkg/store/k8s/layer_stack_group_case_test.go | 67 ++++ .../objectmeta_write_extended_bug_235_test.go | 9 + pkg/store/restore_target.go | 151 +++++++- 12 files changed, 1074 insertions(+), 152 deletions(-) create mode 100644 pkg/store/effective_layer_stack_test.go create mode 100644 pkg/store/k8s/layer_stack_group_case_test.go diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 15c14055..4e46dfad 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -336,7 +336,10 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { return err } - def := newRestoredDefinition(args.toResource, &src, &snap) + def, err := newRestoredDefinition(ctx, run.Store, args.toResource, &src, &snap) + if err != nil { + return fmt.Errorf("restore %s onto %s: %w", snap.Name, args.toResource, err) + } err = run.Store.ResourceDefinitions().Create(ctx, def) if errors.Is(err, store.ErrAlreadyExists) { @@ -365,18 +368,39 @@ func snapshotRestoreResource(ctx context.Context, run *runContext) error { } // newRestoredDefinition is the definition a restore creates: the source's -// group and stack, the snapshot's props, and this restore's marker. -func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *apiv1.Snapshot) *apiv1.ResourceDefinition { +// group and stack, the snapshot's props, and this restore's marker. The stack +// is the one the source's bytes were laid down under +// (store.EffectiveLayerStack); a source whose group has yet to decide it is an +// error the operator retries once the controller pins it. A copy short of what +// its group asks for is flagged the way the controller flags a definition it +// pinned over data. +func newRestoredDefinition( + ctx context.Context, st store.Store, rdName string, src *apiv1.ResourceDefinition, snap *apiv1.Snapshot, +) (*apiv1.ResourceDefinition, error) { + stack, err := store.EffectiveLayerStack(ctx, st, src) + if err != nil { + return nil, fmt.Errorf("the layer stack of the source %s: %w", src.Name, err) + } + + unapplied, err := store.UnappliedGroupStack(ctx, st, src.ResourceGroupName, stack) + if err != nil { + return nil, err //nolint:wrapcheck // names the group itself + } + def := &apiv1.ResourceDefinition{ Name: rdName, // The parent group carries over: it drives later placement and // property inheritance, and a restored definition with a blank // group breaks the restore-then-list workflow. ResourceGroupName: src.ResourceGroupName, - LayerStack: store.EffectiveLayerStack(src.LayerStack), + LayerStack: stack, Props: store.TravellingProps(snap.Props), } + if unapplied != "" { + def.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: unapplied} + } + if def.Props == nil { def.Props = store.TravellingProps(src.Props) } @@ -387,7 +411,7 @@ func newRestoredDefinition(rdName string, src *apiv1.ResourceDefinition, snap *a // the placer looks the source half up as a store key. def.Props = store.WithRestoreMarker(def.Props, snap) - return def + return def, nil } // restoreIntoExistingTarget finishes a restore over a definition that is diff --git a/internal/cli/snapshot_restore_prepared_test.go b/internal/cli/snapshot_restore_prepared_test.go index e55dfefc..35d15231 100644 --- a/internal/cli/snapshot_restore_prepared_test.go +++ b/internal/cli/snapshot_restore_prepared_test.go @@ -1184,20 +1184,54 @@ func TestSnapshotRestorePromotesATieBreakerOnARequestedNode(t *testing.T) { } } -// A source stored without a stack is brought up with the default whatever its -// group says now; the restored definition carries that stack, stamped. -func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) { +func seedUndecidedSnapshotSource(ctx context.Context, backend store.Store) { + seedSnapshotSource(ctx, backend) + _ = backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}, + }) + + // Patched rather than updated: Update keeps a decided stack, and the + // source stands for one stored without a stack. + _ = backend.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, "pvc-x", func(src *apiv1.ResourceDefinition) error { + src.LayerStack = nil + src.ResourceGroupName = "grp-luks" + + return nil + }) +} + +// A source stored without a stack under a group that sets one has its stack +// decided by the controller, which has not done it yet: the restore refuses, +// saying so, and creates nothing. +func TestSnapshotRestoreRefusesAnUndecidedSourceStack(t *testing.T) { + t.Parallel() + + app, _, errBuf := newApp(t, seedUndecidedSnapshotSource) + + if got := app.Run(t.Context(), restoreResourceArgv("pvc-stack")); got == 0 { + t.Fatal("restore of an undecided source exited 0") + } + + if !strings.Contains(errBuf.String(), "not pinned yet") { + t.Errorf("stderr = %q, want it to say the stack is not pinned yet", errBuf.String()) + } + + if _, err := appStore(t, app).ResourceDefinitions().Get(t.Context(), "pvc-stack"); err == nil { + t.Error("the refused restore created the target") + } +} + +// Once the controller pins the source over its data to the default stack, the +// restored definition carries that stack and is flagged as short of its +// group's. +func TestSnapshotRestoreStampsTheSourcesPinnedStack(t *testing.T) { t.Parallel() app, _, errBuf := newApp(t, func(ctx context.Context, backend store.Store) { - seedSnapshotSource(ctx, backend) - _ = backend.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ - Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}, - }) + seedUndecidedSnapshotSource(ctx, backend) src, _ := backend.ResourceDefinitions().Get(ctx, "pvc-x") - src.LayerStack = nil - src.ResourceGroupName = "grp-luks" + src.LayerStack = apiv1.DefaultLayerStack() _ = backend.ResourceDefinitions().Update(ctx, &src) }) @@ -1211,7 +1245,11 @@ func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) { } if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) { - t.Errorf("restored stack = %v, want the source's data-plane %v", rd.LayerStack, apiv1.DefaultLayerStack()) + t.Errorf("restored stack = %v, want the source's %v", rd.LayerStack, apiv1.DefaultLayerStack()) + } + + if got := rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; got != "DRBD,LUKS,STORAGE" { + t.Errorf("unapplied-group annotation = %q, want DRBD,LUKS,STORAGE", got) } } diff --git a/pkg/rest/nodes.go b/pkg/rest/nodes.go index 3ab3b7e7..6cabef58 100644 --- a/pkg/rest/nodes.go +++ b/pkg/rest/nodes.go @@ -1525,6 +1525,18 @@ func storeErrorRc(err error, kind storeResourceKind) (int, apiv1.APICallRc) { RetCode: apiCallRcError | notFoundSubCodeForKind(kind), Message: scrubImplDetails(err.Error()), } + case errors.Is(err, store.ErrLayerStackUndecided): + // Not the caller's to fix and not permanent: the controller pins the + // stack on its next pass over the definition, and linstor-csi retries + // a 503 with the same body. + return http.StatusServiceUnavailable, apiv1.APICallRc{ + RetCode: apiCallRcError, + Message: scrubImplDetails(err.Error()), + Cause: "the definition's volumes are laid down under its group's layer stack or the " + + "default one depending on whether it already holds data, and the controller " + + "has not decided that yet", + Correc: "retry shortly; the controller writes the layer stack onto the definition", + } case apierrors.IsConflict(err): // Optimistic-lock conflict. Generic message — the apimachinery // error embeds the GroupResource string which leaks the CRD diff --git a/pkg/rest/rd_clone.go b/pkg/rest/rd_clone.go index 498a95cb..5eb272b3 100644 --- a/pkg/rest/rd_clone.go +++ b/pkg/rest/rd_clone.go @@ -291,21 +291,79 @@ func (s *Server) cloneRequestShapeIsUsable( } // clonedStackHasItsPassphrase refuses a clone whose stack carries LUKS while -// the cluster passphrase is missing: the stack it names, or the source's when -// it names none. It runs after the replay of a finished clone, which writes -// no new LUKS volume, so a passphrase that went missing or could not be read -// does not fail the retry of a clone that is already done. +// the cluster passphrase is missing: the stack it names, or else the one the +// clone takes from the source, which on the data path is the stack the +// source's bytes were laid down under. It runs after the replay of a finished +// clone, which writes no new LUKS volume, so a passphrase that went missing or +// could not be read does not fail the retry of a clone that is already done. func (s *Server) clonedStackHasItsPassphrase( - ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, withData bool, ) bool { - stack := req.LayerList - if len(stack) == 0 { - stack = store.EffectiveLayerStack(src.LayerStack) + stack := cloneShellLayerStack(src, req) + + if withData && len(req.LayerList) == 0 { + var ok bool + + stack, ok = s.cloneSourceLayerStack(ctx, w, src, req.Name) + if !ok { + return false + } } return s.clonePassphraseHolds(ctx, w, src.Name, req.Name, stack) } +// cloneSourceLayerStack is the stack the source's volumes are laid down +// under (store.EffectiveLayerStack), which a clone carrying its bytes has to +// have. False means a refusal has been written: a source whose stack its +// group has yet to decide answers 503, and a clone waits for the decision +// rather than guess one and copy plaintext into a stack that says LUKS, or +// LUKS into one that says plaintext. +func (s *Server) cloneSourceLayerStack( + ctx context.Context, w http.ResponseWriter, src *apiv1.ResourceDefinition, cloneName string, +) ([]string, bool) { + stack, err := store.EffectiveLayerStack(ctx, s.Store, src) + if err != nil { + status, callRc := storeErrorRc(err, storeKindResourceDfn) + callRc.Message = "clone of resource definition '" + src.Name + "': " + callRc.Message + writeCloneRefused(w, status, src.Name, cloneName, &callRc) + + return nil, false + } + + return stack, true +} + +// withoutUnappliedGroupStack is a copy of annotations without the flag the +// controller puts on a definition pinned over data. A copy, because the map +// the store hands back can be the stored object's own. +func withoutUnappliedGroupStack(annotations map[string]string) map[string]string { + out := maps.Clone(annotations) + delete(out, apiv1.UnappliedGroupLayerStackAnnotation) + + return out +} + +// cloneShellLayerStack is the stack a volume-less clone is created with: the +// caller's, or else the source's as stored. A shell holds no bytes, so nothing +// forces a stack on it the way the data path is forced; left empty, it takes +// its own group's stack from the controller, which pins it once +// (pinLayerStack), exactly like a definition created empty. A source whose +// stack was pinned over data that was already there, which the +// unapplied-group annotation marks, passes that stack on to no shell: it was +// the data's, not a choice. +func cloneShellLayerStack(src *apiv1.ResourceDefinition, req *rdCloneRequest) []string { + if len(req.LayerList) > 0 { + return req.LayerList + } + + if src.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] != "" { + return nil + } + + return slices.Clone(src.LayerStack) +} + // clonePassphraseHolds refuses a clone whose stack carries LUKS while the // cluster passphrase is missing. False means a refusal has been written. // @@ -439,10 +497,6 @@ func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { return } - if !s.clonedStackHasItsPassphrase(r.Context(), w, &src, &req) { - return - } - // VD-bearing sources take the snapshot-based data-plane path // (Bug-020); vol-less sources keep the legacy shallow-copy // contract Group D pins. @@ -456,6 +510,10 @@ func (s *Server) handleRDClone(w http.ResponseWriter, r *http.Request) { return } + if !s.clonedStackHasItsPassphrase(r.Context(), w, &src, &req, len(srcVDs) > 0) { + return + } + if len(srcVDs) > 0 { s.cloneWithData(w, r, &src, &req) @@ -548,7 +606,12 @@ func (s *Server) cloneWithData(w http.ResponseWriter, r *http.Request, src *apiv return } - if !cloneLayerStackIsHonourable(w, src, req) { + have, ok := s.cloneSourceLayerStack(ctx, w, src, req.Name) + if !ok { + return + } + + if !cloneLayerStackIsHonourable(w, src, req, have) { return } @@ -1013,21 +1076,18 @@ func uncheckedCloneGroupWarning(cloneName, rgName string, err error) *apiv1.APIC // Only the data-bearing path calls this. A clone of a VD-less source carries // no bytes to lose, and choosing a different stack for the shell is what // accepting layer_list is for. -func cloneLayerStackIsHonourable(w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest) bool { +func cloneLayerStackIsHonourable( + w http.ResponseWriter, src *apiv1.ResourceDefinition, req *rdCloneRequest, have []string, +) bool { if len(req.LayerList) == 0 { return true } - // An RD stored without an explicit stack is not a definition with no - // layers — it is one that never said, and every other reader resolves - // that to apiv1.DefaultLayerStack (stampRDLayerDataFromStack does it on - // the read path). Resolving it here too is what keeps the gate off the - // CSI hot path: linstor-csi sends [DRBD, STORAGE] on every clone, and - // against a bare source an unresolved empty stack would read as "adds - // DRBD, STORAGE" and refuse the clone this endpoint was just unbroken - // for. - have := store.EffectiveLayerStack(src.LayerStack) - + // have is the source's stack as its volumes are laid down + // (store.EffectiveLayerStack), never the raw field: an empty one there is + // a definition that never said, and read unresolved against the + // [DRBD, STORAGE] linstor-csi sends on every clone it would read as + // "adds DRBD, STORAGE" and refuse the clone this endpoint exists for. added, dropped := layerSetDifference(have, req.LayerList) if len(added) == 0 && len(dropped) == 0 { return true @@ -1584,7 +1644,14 @@ func (s *Server) replayOfFinishedClone( // It runs before anything established whether the leftover is finished, // so its wording is the resume's: the leftover was started with a shape, // whether or not it got further. - if differs := requestedShapeDiffers(&existing, req.ResourceGroup, req.LayerList); differs != "" { + differs, err := s.requestedShapeDiffers(ctx, &existing, req.ResourceGroup, req.LayerList) + if err != nil { + writeCloneStoreError(w, srcName, cloneName, err) + + return false, true + } + + if differs != "" { writeCloneRefused(w, http.StatusConflict, srcName, cloneName, &apiv1.APICallRc{ RetCode: apiCallRcError, Message: "clone target '" + cloneName + "' was started with " + differs, @@ -2109,15 +2176,11 @@ func (s *Server) cloneEmptyRDShell(w http.ResponseWriter, r *http.Request, // inherited them would carry a delete, or any later flag, it never had. clone.Flags = nil - // The caller's own shape wins over the source's, on both clone paths. - // Without one the shell is stamped with the source's data-plane stack, as - // the data path stamps its target: left empty under a target group of - // another stack, the control plane would judge it by that group while the - // satellite brings it up as the default, once volumes are added. - clone.LayerStack = store.EffectiveLayerStack(src.LayerStack) - if len(req.LayerList) > 0 { - clone.LayerStack = req.LayerList - } + // The caller's own shape wins over the source's, on both clone paths; + // see cloneShellLayerStack for the rest. The flag the controller puts on a + // definition pinned over data stays with that data: the shell has none. + clone.LayerStack = cloneShellLayerStack(src, req) + clone.Annotations = withoutUnappliedGroupStack(src.Annotations) if req.ResourceGroup != "" { clone.ResourceGroupName = req.ResourceGroup diff --git a/pkg/rest/rd_clone_validation_test.go b/pkg/rest/rd_clone_validation_test.go index f13e1d81..460d4bcb 100644 --- a/pkg/rest/rd_clone_validation_test.go +++ b/pkg/rest/rd_clone_validation_test.go @@ -176,6 +176,44 @@ func TestRDCloneOfALUKSSourceNeedsThePassphraseWithoutALayerList(t *testing.T) { } } +// A LUKS source flagged as short of its group's stack passes its stack on +// to no volume-less shell, but a clone of its volumes carries its encrypted +// bytes and takes that stack all the same. Without the passphrase that is +// refused, not written as a LUKS volume the satellite cannot open. +func TestRDCloneOfAFlaggedLUKSSourceWithDataNeedsThePassphrase(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedCloneSourceWithStack(t, st, "src-flagged", []string{"DRBD", "LUKS", "STORAGE"}) + + src, err := st.ResourceDefinitions().Get(t.Context(), "src-flagged") + if err != nil { + t.Fatalf("read the seeded source: %v", err) + } + + src.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: "DRBD,LUKS,CACHE,STORAGE"} + + if err := st.ResourceDefinitions().Update(t.Context(), &src); err != nil { + t.Fatalf("flag the source: %v", err) + } + + base, stop := startServerCustom(t, &Server{ + Addr: pickFreeAddr(t), + Store: st, + Client: newFakeRESTClient(t), + Namespace: testRESTNamespace, + }) + defer stop() + + if code := cloneOnce(t, base, "src-flagged", "dst-flagged", nil); code != http.StatusBadRequest { + t.Errorf("clone with data of a flagged LUKS source and no passphrase = %d, want 400", code) + } + + if _, err := st.ResourceDefinitions().Get(t.Context(), "dst-flagged"); err == nil { + t.Error("the refused clone wrote its target") + } +} + // The replay of a finished LUKS clone writes no new LUKS volume, so a // passphrase Secret that cannot be read at that moment does not fail it: a // driver that lost the first answer would otherwise be refused a clone that diff --git a/pkg/rest/restore_effective_layers_test.go b/pkg/rest/restore_effective_layers_test.go index cdf9470f..f097dbe7 100644 --- a/pkg/rest/restore_effective_layers_test.go +++ b/pkg/rest/restore_effective_layers_test.go @@ -21,29 +21,72 @@ func seedLayeredGroup(t *testing.T, st store.Store, name string, stack []string) } } -func assertDataPlaneStack(t *testing.T, st store.Store, rdName string) { +func luksStack() []string { return []string{"DRBD", "LUKS", "STORAGE"} } + +func readRD(t *testing.T, st store.Store, name string) apiv1.ResourceDefinition { t.Helper() - rd, err := st.ResourceDefinitions().Get(t.Context(), rdName) + rd, err := st.ResourceDefinitions().Get(t.Context(), name) if err != nil { - t.Fatalf("read %q: %v", rdName, err) + t.Fatalf("read %q: %v", name, err) } - if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) { - t.Errorf("%q stack = %v, want the source's %v stamped: left empty, the control plane "+ - "would judge it by its group's stack while the satellite brings it up as the source's", - rdName, rd.LayerStack, apiv1.DefaultLayerStack()) + return rd +} + +// reshapeRD sets a definition's group, stack and annotations the way the +// controller pin or an operator edit would. +func reshapeRD(t *testing.T, st store.Store, name, group string, stack []string, annotations map[string]string) { + t.Helper() + + // Patched rather than updated: Update keeps a decided stack, and an + // empty one here stands for a definition an older controller stored + // without one. + err := st.ResourceDefinitions().PatchResourceDefinitionSpec(t.Context(), name, func(rd *apiv1.ResourceDefinition) error { + rd.ResourceGroupName = group + rd.LayerStack = stack + rd.Annotations = annotations + + return nil + }) + if err != nil { + t.Fatalf("reshape %q: %v", name, err) + } +} + +func assertRDAbsent(t *testing.T, st store.Store, name string) { + t.Helper() + + if _, err := st.ResourceDefinitions().Get(t.Context(), name); err == nil { + t.Errorf("%q exists after a refusal that should have written nothing", name) + } +} + +func assertStack(t *testing.T, st store.Store, name string, want []string) { + t.Helper() + + if got := readRD(t, st, name).LayerStack; !slices.Equal(got, want) { + t.Errorf("%q stack = %v, want %v", name, got, want) + } +} + +func assertUnapplied(t *testing.T, st store.Store, name, want string) { + t.Helper() + + if got := readRD(t, st, name).Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; got != want { + t.Errorf("%q unapplied-group annotation = %q, want %q", name, got, want) } } // A clone named into a group whose stack differs keeps the stack the source's -// bytes were written under, stamped, so the group's stack cannot reach it. +// bytes were written under, stamped, so the group's stack cannot reach it, and +// it is flagged as short of what its group asks for. func TestRDCloneIntoAGroupOfAnotherStackKeepsTheSourcesStack(t *testing.T) { t.Parallel() st := store.NewInMemory() seedDeployedCloneSource(t, st, "src-grp") - seedLayeredGroup(t, st, "grp-luks", []string{"DRBD", "LUKS", "STORAGE"}) + seedLayeredGroup(t, st, "grp-luks", luksStack()) base, stop := startServerWithStore(t, st) defer stop() @@ -52,32 +95,184 @@ func TestRDCloneIntoAGroupOfAnotherStackKeepsTheSourcesStack(t *testing.T) { t.Fatalf("clone into a group of another stack = %d, want 201", code) } - assertDataPlaneStack(t, st, "dst-grp") + assertStack(t, st, "dst-grp", apiv1.DefaultLayerStack()) + assertUnapplied(t, st, "dst-grp", "DRBD,LUKS,STORAGE") } -// A volume-less source is cloned as a shell, and the shell is stamped with the -// source's data-plane stack the same way the data path stamps its target: left -// empty under a group of another stack, volumes added to it later would be -// brought up as the default while the control plane judged them by the group. -func TestRDCloneShellIntoAGroupOfAnotherStackKeepsTheSourcesStack(t *testing.T) { +// A clone of a source whose stack its group has yet to decide waits for the +// decision: 503, nothing written. Once the controller pins the source (here +// over data, so to the default stack), the same request clones it with that +// stack. +func TestRDCloneOfAnUndecidedSourceWaitsForTheStack(t *testing.T) { t.Parallel() st := store.NewInMemory() - seedLayeredGroup(t, st, "grp-shell-luks", []string{"DRBD", "LUKS", "STORAGE"}) + ctx := t.Context() + seedDeployedCloneSource(t, st, "src-und") + seedLayeredGroup(t, st, "grp-und", luksStack()) + reshapeRD(t, st, "src-und", "grp-und", nil, nil) + + base, stop := startServerWithStore(t, st) + defer stop() + + for _, extra := range []map[string]any{nil, {"layer_list": []string{"DRBD", "STORAGE"}}} { + if code := cloneOnce(t, base, "src-und", "dst-und", extra); code != http.StatusServiceUnavailable { + t.Errorf("clone of an undecided source (%v) = %d, want 503", extra, code) + } + } + + assertRDAbsent(t, st, "dst-und") + + if _, err := st.Snapshots().Get(ctx, "src-und", cloneSnapshotName("dst-und")); err == nil { + t.Error("the refused clone took its internal snapshot of the source") + } + + reshapeRD(t, st, "src-und", "grp-und", apiv1.DefaultLayerStack(), + map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: "DRBD,LUKS,STORAGE"}) + + if code := cloneOnce(t, base, "src-und", "dst-und", nil); code != http.StatusCreated { + t.Fatalf("clone once the source is pinned = %d, want 201", code) + } + + assertStack(t, st, "dst-und", apiv1.DefaultLayerStack()) + assertUnapplied(t, st, "dst-und", "DRBD,LUKS,STORAGE") +} - if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{Name: "src-shell"}); err != nil { +// A finished clone is answered before the source is read, so a source that +// became undecided since (moved under a group with a stack) does not fail its +// replay. +func TestRDCloneReplayDoesNotReadTheSourcesStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-rep") + seedLayeredGroup(t, st, "grp-rep", luksStack()) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-rep", "dst-rep", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + reshapeRD(t, st, "src-rep", "grp-rep", nil, nil) + + if code := cloneOnce(t, base, "src-rep", "dst-rep", nil); code != http.StatusCreated { + t.Errorf("replay over a source that became undecided = %d, want 201", code) + } + + if code := cloneOnce(t, base, "src-rep", "dst-rep", + map[string]any{"layer_list": []string{"DRBD", "STORAGE"}}); code != http.StatusCreated { + t.Errorf("replay naming the stack the clone has = %d, want 201", code) + } +} + +// A finished clone left with an empty stack, as an older controller wrote +// them, is compared against a named stack once its group decides it: 503 +// until the controller pins it, the usual answer after. +func TestRDCloneReplayOfAnUndecidedLeftoverWaitsForTheStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-old") + seedLayeredGroup(t, st, "grp-old", luksStack()) + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-old", "dst-old", nil); code != http.StatusCreated { + t.Fatalf("first clone = %d, want 201", code) + } + + dst := readRD(t, st, "dst-old") + reshapeRD(t, st, "dst-old", "grp-old", nil, dst.Annotations) + + named := map[string]any{"layer_list": []string{"DRBD", "STORAGE"}} + if code := cloneOnce(t, base, "src-old", "dst-old", named); code != http.StatusServiceUnavailable { + t.Errorf("replay over an undecided leftover = %d, want 503", code) + } + + reshapeRD(t, st, "dst-old", "grp-old", apiv1.DefaultLayerStack(), dst.Annotations) + + if code := cloneOnce(t, base, "src-old", "dst-old", named); code != http.StatusCreated { + t.Errorf("replay once the leftover is pinned = %d, want 201", code) + } +} + +// A volume-less source that never said what stack it has passes that on: the +// shell holds no bytes, so its own group decides through the controller pin, +// as for any definition created empty. +func TestRDCloneShellOfASourceThatNeverSaidLeavesTheStackToItsGroup(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedLayeredGroup(t, st, "grp-shell-luks", luksStack()) + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-shell", ResourceGroupName: "grp-shell-luks", + }); err != nil { t.Fatalf("seed the volume-less source: %v", err) } base, stop := startServerWithStore(t, st) defer stop() - if code := cloneOnce(t, base, "src-shell", "dst-shell", - map[string]any{"resource_group": "grp-shell-luks"}); code != http.StatusCreated { - t.Fatalf("shell clone into a group of another stack = %d, want 201", code) + if code := cloneOnce(t, base, "src-shell", "dst-shell", nil); code != http.StatusCreated { + t.Fatalf("shell clone = %d, want 201", code) } - assertDataPlaneStack(t, st, "dst-shell") + assertStack(t, st, "dst-shell", nil) + assertUnapplied(t, st, "dst-shell", "") +} + +// A stack the controller pinned over data that was already there is the data's, +// not a choice: a shell, which holds none, does not take it or its flag. +func TestRDCloneShellDoesNotInheritAStackPinnedOverData(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedLayeredGroup(t, st, "grp-pinned", luksStack()) + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-pinned", ResourceGroupName: "grp-pinned", LayerStack: apiv1.DefaultLayerStack(), + Annotations: map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: "DRBD,LUKS,STORAGE"}, + }); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-pinned", "dst-pinned", nil); code != http.StatusCreated { + t.Fatalf("shell clone = %d, want 201", code) + } + + assertStack(t, st, "dst-pinned", nil) + assertUnapplied(t, st, "dst-pinned", "") + assertUnapplied(t, st, "src-pinned", "DRBD,LUKS,STORAGE") +} + +// A stack the source chose is copied onto its shell. +func TestRDCloneShellKeepsAStackTheSourceChose(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedLayeredGroup(t, st, "grp-chose", luksStack()) + + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-chose", ResourceGroupName: "grp-chose", LayerStack: apiv1.DefaultLayerStack(), + }); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithStore(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-chose", "dst-chose", nil); code != http.StatusCreated { + t.Fatalf("shell clone = %d, want 201", code) + } + + assertStack(t, st, "dst-chose", apiv1.DefaultLayerStack()) } // The shell clone stores the stack the caller named in canonical case too. @@ -97,85 +292,127 @@ func TestRDCloneShellStoresALowercaseLayerListCanonically(t *testing.T) { t.Fatalf("shell clone with a lowercase layer_list = %d, want 201", code) } - dst, err := st.ResourceDefinitions().Get(t.Context(), "dst-case") - if err != nil { - t.Fatalf("read the clone: %v", err) - } - - if !slices.Equal(dst.LayerStack, []string{"DRBD", "STORAGE"}) { - t.Errorf("layer stack = %v, want [DRBD STORAGE]", dst.LayerStack) - } + assertStack(t, st, "dst-case", []string{"DRBD", "STORAGE"}) } -// A restore inherits the source's group, whose stack may since differ from -// the one the source was created with. -func TestSnapshotRestoreStampsTheSourcesDataPlaneStack(t *testing.T) { +// A restore of a source whose stack its group has yet to decide waits for the +// decision: 503, nothing written. Once the controller pins the source over its +// data, the restore stamps that stack on the new definition and flags it as +// short of its group's. +func TestSnapshotRestoreWaitsForTheSourcesStackToBeDecided(t *testing.T) { t.Parallel() st := store.NewInMemory() - ctx := t.Context() seedDeployedCloneSource(t, st, "src-rst") - seedLayeredGroup(t, st, "grp-changed", []string{"DRBD", "LUKS", "STORAGE"}) + seedLayeredGroup(t, st, "grp-changed", luksStack()) + reshapeRD(t, st, "src-rst", "grp-changed", nil, nil) + seedRestoreSnapshot(t, st, "src-rst", "snap-rst", []string{"node-a"}) - src, err := st.ResourceDefinitions().Get(ctx, "src-rst") - if err != nil { - t.Fatalf("read the source: %v", err) - } + base, stop := startServerWithStore(t, st) + defer stop() - src.ResourceGroupName = "grp-changed" - if err := st.ResourceDefinitions().Update(ctx, &src); err != nil { - t.Fatalf("move the source under the changed group: %v", err) + if code := restoreOnce(t, base, "src-rst", "snap-rst", map[string]any{"to_resource": "dst-rst"}); code != http.StatusServiceUnavailable { + t.Fatalf("restore of an undecided source = %d, want 503", code) } - seedRestoreSnapshot(t, st, "src-rst", "snap-rst", []string{"node-a"}) + assertRDAbsent(t, st, "dst-rst") - base, stop := startServerWithStore(t, st) - defer stop() + reshapeRD(t, st, "src-rst", "grp-changed", apiv1.DefaultLayerStack(), nil) if code := restoreOnce(t, base, "src-rst", "snap-rst", map[string]any{"to_resource": "dst-rst"}); code != http.StatusCreated { - t.Fatalf("restore = %d, want 201", code) + t.Fatalf("restore once the source is pinned = %d, want 201", code) } - assertDataPlaneStack(t, st, "dst-rst") + assertStack(t, st, "dst-rst", apiv1.DefaultLayerStack()) + assertUnapplied(t, st, "dst-rst", "DRBD,LUKS,STORAGE") } -// A prepared target stored without a stack is brought up with the default, -// whatever its group says, and is judged and stamped by that. -func TestSnapshotRestoreIntoAPreparedTargetStampsItsDataPlaneStack(t *testing.T) { +// A group that cannot be read leaves the source's stack unknown, and the +// restore refuses with a retryable answer before it writes anything. +func TestSnapshotRestoreRefusesWhenTheSourcesGroupCannotBeRead(t *testing.T) { t.Parallel() - st := store.NewInMemory() + backend := store.NewInMemory() + seedDeployedCloneSource(t, backend, "src-unread") + seedLayeredGroup(t, backend, "grp-unread", luksStack()) + reshapeRD(t, backend, "src-unread", "grp-unread", nil, nil) + seedRestoreSnapshot(t, backend, "src-unread", "snap-unread", []string{"node-a"}) + + base, stop := startServerWithStore(t, failingRGReadStore{backend}) + defer stop() + + code := restoreOnce(t, base, "src-unread", "snap-unread", map[string]any{"to_resource": "dst-unread"}) + if code != http.StatusInternalServerError { + t.Errorf("restore over an unreadable group = %d, want 500", code) + } + + assertRDAbsent(t, backend, "dst-unread") +} + +func seedPreparedTarget(t *testing.T, st store.Store, srcName, snapName, rdName, group string, stack []string) { + t.Helper() + ctx := t.Context() - seedDeployedCloneSource(t, st, "src-prep") - seedRestoreSnapshot(t, st, "src-prep", "snap-prep", []string{"node-a"}) - seedLayeredGroup(t, st, "grp-sc", []string{"DRBD", "LUKS", "STORAGE"}) - snap, err := st.Snapshots().Get(ctx, "src-prep", "snap-prep") + snap, err := st.Snapshots().Get(ctx, srcName, snapName) if err != nil { t.Fatalf("read the snapshot: %v", err) } if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ - Name: "pvc-prep", ResourceGroupName: "grp-sc", + Name: rdName, ResourceGroupName: group, LayerStack: stack, }); err != nil { t.Fatalf("seed the prepared target: %v", err) } for _, svd := range snap.VolumeDefinitions { - if err := st.VolumeDefinitions().Create(ctx, "pvc-prep", + if err := st.VolumeDefinitions().Create(ctx, rdName, &apiv1.VolumeDefinition{VolumeNumber: svd.VolumeNumber, SizeKib: svd.SizeKib}); err != nil { t.Fatalf("seed the prepared volume: %v", err) } } +} + +// A prepared target stored without a stack under a group that sets one waits +// for the controller to decide it: 503, and no marker goes on. Pinned to the +// group's LUKS stack, as a target with nothing on it is, it is refused against +// a plaintext source; one the operator gave the source's stack is restored +// into and keeps it. +func TestSnapshotRestoreIntoAnUndecidedPreparedTargetWaits(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedDeployedCloneSource(t, st, "src-prep") + seedRestoreSnapshot(t, st, "src-prep", "snap-prep", []string{"node-a"}) + seedLayeredGroup(t, st, "grp-sc", luksStack()) + seedPreparedTarget(t, st, "src-prep", "snap-prep", "pvc-prep", "grp-sc", nil) base, stop := startServerWithStore(t, st) defer stop() - if code := restoreOnce(t, base, "src-prep", "snap-prep", map[string]any{ - "to_resource": "pvc-prep", "nodes": []string{"node-a"}, - }); code != http.StatusCreated { - t.Fatalf("restore into the prepared target = %d, want 201", code) + body := map[string]any{"to_resource": "pvc-prep", "nodes": []string{"node-a"}} + + if code := restoreOnce(t, base, "src-prep", "snap-prep", body); code != http.StatusServiceUnavailable { + t.Fatalf("restore into an undecided prepared target = %d, want 503", code) + } + + if marker := readRD(t, st, "pvc-prep").Props[store.RestoreFromSnapshotProp]; marker != "" { + t.Errorf("the refused restore left marker %q on the target", marker) + } + + assertStack(t, st, "pvc-prep", nil) + + reshapeRD(t, st, "pvc-prep", "grp-sc", luksStack(), nil) + + if code := restoreOnce(t, base, "src-prep", "snap-prep", body); code != http.StatusConflict { + t.Errorf("restore of a plaintext source into a LUKS target = %d, want 409", code) + } + + reshapeRD(t, st, "pvc-prep", "grp-sc", apiv1.DefaultLayerStack(), nil) + + if code := restoreOnce(t, base, "src-prep", "snap-prep", body); code != http.StatusCreated { + t.Fatalf("restore into a target of the source's stack = %d, want 201", code) } - assertDataPlaneStack(t, st, "pvc-prep") + assertStack(t, st, "pvc-prep", apiv1.DefaultLayerStack()) } diff --git a/pkg/rest/rg_deleted_race_test.go b/pkg/rest/rg_deleted_race_test.go index 104acaaf..787dfb6b 100644 --- a/pkg/rest/rg_deleted_race_test.go +++ b/pkg/rest/rg_deleted_race_test.go @@ -339,6 +339,36 @@ func (f failingRGReadStore) ResourceGroups() store.ResourceGroupStore { return failingRGReads{f.Store.ResourceGroups()} } +// failingRGRecheckStore fails the group reads that come after the target +// definition exists, which is the post-write re-check. The reads before the +// write, the one that resolves the source's layer stack among them, decide +// what gets written, so a failure there refuses before anything lands and is +// not what these tests are about. +type failingRGRecheckStore struct { + store.Store + + target string +} + +func (f failingRGRecheckStore) ResourceGroups() store.ResourceGroupStore { + return failingRGRecheckReads{ResourceGroupStore: f.Store.ResourceGroups(), backend: f.Store, target: f.target} +} + +type failingRGRecheckReads struct { + store.ResourceGroupStore + + backend store.Store + target string +} + +func (f failingRGRecheckReads) Get(ctx context.Context, name string) (apiv1.ResourceGroup, error) { + if _, err := f.backend.ResourceDefinitions().Get(ctx, f.target); err == nil { + return apiv1.ResourceGroup{}, errRGReadFailed + } + + return f.ResourceGroupStore.Get(ctx, name) //nolint:wrapcheck // test double +} + // The post-write check is a safety net over a restore that already succeeded. // When the net itself cannot be inspected, undoing the restore trades a rare // dangling parent group for a certain lost restore — and a worse one, because @@ -375,7 +405,7 @@ func TestSnapshotRestoreSurvivesAFailedParentGroupRecheck(t *testing.T) { t.Fatalf("seed the snapshot: %v", err) } - base, stop := startServerWithStore(t, failingRGReadStore{backend}) + base, stop := startServerWithStore(t, failingRGRecheckStore{Store: backend, target: "flaky-dst"}) defer stop() body, _ := json.Marshal(map[string]string{ @@ -504,7 +534,7 @@ func TestRDCloneSurvivesAFailedParentGroupRecheck(t *testing.T) { ctx := t.Context() seedGroupedCloneSource(t, backend, "src-flaky-rg", "grp-flaky-clone", true) - base, stop := startServerWithStore(t, failingRGReadStore{backend}) + base, stop := startServerWithStore(t, failingRGRecheckStore{Store: backend, target: "dst-flaky-rg"}) defer stop() resp := postClone(t, base, "src-flaky-rg", map[string]any{ diff --git a/pkg/rest/snapshot_restore.go b/pkg/rest/snapshot_restore.go index f96ff70d..fa32b1da 100644 --- a/pkg/rest/snapshot_restore.go +++ b/pkg/rest/snapshot_restore.go @@ -1102,7 +1102,9 @@ func resolveSnapshotName(r *http.Request, req *snapshotRestoreRequest) string { // marker says the definition is this operation's own, the DELETE flag says // whether it is still there to finish, and the fields the caller named say // whether it is the same operation. -func leftoverIsThisRestore(existing *apiv1.ResourceDefinition, snap *apiv1.Snapshot, overrides *rdShapeOverrides) bool { +func (s *Server) leftoverIsThisRestore( + ctx context.Context, existing *apiv1.ResourceDefinition, snap *apiv1.Snapshot, overrides *rdShapeOverrides, +) (bool, error) { var namedRG string var namedLayers []string @@ -1111,9 +1113,14 @@ func leftoverIsThisRestore(existing *apiv1.ResourceDefinition, snap *apiv1.Snaps namedRG, namedLayers = overrides.ResourceGroupName, overrides.LayerStack } - return restoreMarkerMatches(existing.Props, snap.ResourceName, snap.Name) && - !slices.Contains(existing.Flags, rdFlagDelete) && - requestedShapeDiffers(existing, namedRG, namedLayers) == "" + if !restoreMarkerMatches(existing.Props, snap.ResourceName, snap.Name) || + slices.Contains(existing.Flags, rdFlagDelete) { + return false, nil + } + + differs, err := s.requestedShapeDiffers(ctx, existing, namedRG, namedLayers) + + return differs == "" && err == nil, err } // requestedShapeDiffers names the first way a definition already under the @@ -1140,29 +1147,36 @@ func leftoverIsThisRestore(existing *apiv1.ResourceDefinition, snap *apiv1.Snaps // group materializeRestoredRD stamped, the source's unless the request named // one, so without it a request that names no group is compared as "" against // that group and refused. -func requestedShapeDiffers(existing *apiv1.ResourceDefinition, rgName string, layers []string) string { +func (s *Server) requestedShapeDiffers( + ctx context.Context, existing *apiv1.ResourceDefinition, rgName string, layers []string, +) (string, error) { if rgName != "" && !strings.EqualFold(existing.ResourceGroupName, rgName) { - return "resource group '" + existing.ResourceGroupName + "', not '" + rgName + "'" + return "resource group '" + existing.ResourceGroupName + "', not '" + rgName + "'", nil } if len(layers) == 0 { - return "" + return "", nil + } + + // The leftover's stack as its volumes are laid down, not the raw field. A + // leftover materializeRestoredRD wrote carries its stack explicitly; one + // left empty by an older controller is resolved through its group, and + // is undecided until the controller pins it, which a retry outlasts. + // Unresolved, an empty stack compared against a retry naming + // [DRBD, STORAGE], which is every linstor-csi retry, reads as adding both + // layers and refuses the resume. A retry that names no stack is not + // compared at all. + have, err := store.EffectiveLayerStack(ctx, s.Store, existing) + if err != nil { + return "", errors.Wrapf(err, "the layer stack of %q", existing.Name) } - // An unset stack is a definition that never said, which is what - // materializeRestoredRD copies off a source that never said either, and - // not a definition with no layers. Unresolved, that leftover compared - // against a retry naming [DRBD, STORAGE], which is every linstor-csi - // retry, reads as adding both layers and refuses the resume. A retry that - // names no stack is not compared at all. - have := store.EffectiveLayerStack(existing.LayerStack) - added, dropped := layerSetDifference(have, layers) if len(added) > 0 || len(dropped) > 0 { - return "layer stack " + strings.Join(have, ",") + ", not " + strings.Join(layers, ",") + return "layer stack " + strings.Join(have, ",") + ", not " + strings.Join(layers, ","), nil } - return "" + return "", nil } // uncheckedRestoreGroupWarning tells the caller the restore worked and that @@ -1336,6 +1350,15 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s return materialisedRD{}, err //nolint:wrapcheck // surfaced via writeStoreError } + // The restored bytes were laid down under the source's stack, so that is + // the target's, unless a clone names its own (which its gate holds to the + // same set). A source whose group has yet to decide it is refused until + // the controller does, rather than guessed. + stack, err := store.EffectiveLayerStack(ctx, s.Store, &srcRDObj) + if err != nil { + return materialisedRD{}, err //nolint:wrapcheck // surfaced via writeStoreError + } + newRD := apiv1.ResourceDefinition{ Name: req.ToResource, // Bug 151: carry over the source's ResourceGroupName so the @@ -1347,7 +1370,7 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s // auto-placement and prop inheritance). ResourceGroupName: srcRDObj.ResourceGroupName, Props: store.TravellingProps(snap.Props), - LayerStack: srcRDObj.LayerStack, + LayerStack: stack, } if newRD.Props == nil { @@ -1356,11 +1379,13 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s overrides.applyTo(&newRD) - // The stack the restored bytes are brought up under is the source's as - // the data plane resolves it. Left empty, a group named for the target - // whose stack differs would have the control plane judge the definition - // by that group's stack while the satellite brings it up as the source's. - newRD.LayerStack = store.EffectiveLayerStack(newRD.LayerStack) + // A copy short of what its group asks for, a plaintext one in a LUKS + // group, is flagged the way the controller flags a definition it pinned + // over data; see store.UnappliedGroupStack. + err = flagUnappliedGroupStack(ctx, s.Store, &newRD) + if err != nil { + return materialisedRD{}, err + } // Stamp the clone-source so the dispatcher's buildVolumes (called // at every satellite-reconcile of placed Resources) emits @@ -1403,6 +1428,27 @@ func (s *Server) materializeRestoredRD(ctx context.Context, srcRD string, req *s return made, nil } +// flagUnappliedGroupStack annotates rd with the layers its group asks for and +// its own stack lacks, when there are any (store.UnappliedGroupStack). +func flagUnappliedGroupStack(ctx context.Context, st store.Store, rd *apiv1.ResourceDefinition) error { + unapplied, err := store.UnappliedGroupStack(ctx, st, rd.ResourceGroupName, rd.LayerStack) + if err != nil { + return err //nolint:wrapcheck // surfaced via writeStoreError + } + + if unapplied == "" { + return nil + } + + if rd.Annotations == nil { + rd.Annotations = map[string]string{} + } + + rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] = unapplied + + return nil +} + // createOrAdoptRestoredRD creates the restored definition, and reports whether // this request created it and, when it adopted one instead, the group that // leftover carries. @@ -1438,7 +1484,12 @@ func (s *Server) createOrAdoptRestoredRD(ctx context.Context, newRD *apiv1.Resou // between the state check above and this Create — but it is the exact // state the 409 in restoreTargetState exists to prevent, and hydrating // volumes into a dying definition races the tear-down reaping them. - if !leftoverIsThisRestore(&existing, snap, overrides) { + own, ownErr := s.leftoverIsThisRestore(ctx, &existing, snap, overrides) + if ownErr != nil { + return false, "", ownErr + } + + if !own { return false, "", err //nolint:wrapcheck // surfaced via writeStoreError } diff --git a/pkg/store/effective_layer_stack_test.go b/pkg/store/effective_layer_stack_test.go new file mode 100644 index 00000000..8f58ac72 --- /dev/null +++ b/pkg/store/effective_layer_stack_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 + +package store_test + +import ( + "context" + "errors" + "slices" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" +) + +var errGroupUnreadable = errors.New("probe: the group could not be read") + +type unreadableGroups struct { + store.ResourceGroupStore +} + +func (unreadableGroups) Get(context.Context, string) (apiv1.ResourceGroup, error) { + return apiv1.ResourceGroup{}, errGroupUnreadable +} + +type unreadableGroupStore struct { + store.Store +} + +func (s unreadableGroupStore) ResourceGroups() store.ResourceGroupStore { + return unreadableGroups{s.Store.ResourceGroups()} +} + +func seedGroups(t *testing.T, st store.Store) { + t.Helper() + + for _, rg := range []apiv1.ResourceGroup{ + {Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}}, + {Name: "grp-plain"}, + } { + if err := st.ResourceGroups().Create(t.Context(), &rg); err != nil { + t.Fatalf("seed group %q: %v", rg.Name, err) + } + } +} + +// The stack a definition's volumes are laid down under: its own; the default +// for an empty one with nothing above it that sets one; and undecided for an +// empty one under a group that sets one, since the controller decides that by +// whether data is already there. +func TestEffectiveLayerStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedGroups(t, st) + + for _, tc := range []struct { + name string + rd apiv1.ResourceDefinition + want []string + undec bool + }{ + {"own stack", apiv1.ResourceDefinition{Name: "a", ResourceGroupName: "grp-luks", LayerStack: []string{"DRBD", "STORAGE"}}, []string{"DRBD", "STORAGE"}, false}, + {"no group", apiv1.ResourceDefinition{Name: "b"}, apiv1.DefaultLayerStack(), false}, + {"group gone", apiv1.ResourceDefinition{Name: "c", ResourceGroupName: "grp-gone"}, apiv1.DefaultLayerStack(), false}, + {"group sets none", apiv1.ResourceDefinition{Name: "d", ResourceGroupName: "grp-plain"}, apiv1.DefaultLayerStack(), false}, + {"group sets one", apiv1.ResourceDefinition{Name: "e", ResourceGroupName: "grp-luks"}, nil, true}, + } { + got, err := store.EffectiveLayerStack(t.Context(), st, &tc.rd) + if tc.undec { + if !errors.Is(err, store.ErrLayerStackUndecided) { + t.Errorf("%s: err = %v, want ErrLayerStackUndecided", tc.name, err) + } + + continue + } + + if err != nil || !slices.Equal(got, tc.want) { + t.Errorf("%s: = %v, %v; want %v", tc.name, got, err, tc.want) + } + } +} + +// A group that cannot be read is an error, never the default and never +// undecided: the caller cannot tell, so it retries. +func TestEffectiveLayerStackDoesNotGuessOverAnUnreadableGroup(t *testing.T) { + t.Parallel() + + backend := store.NewInMemory() + seedGroups(t, backend) + + rd := apiv1.ResourceDefinition{Name: "f", ResourceGroupName: "grp-plain"} + + _, err := store.EffectiveLayerStack(t.Context(), unreadableGroupStore{backend}, &rd) + if !errors.Is(err, errGroupUnreadable) || errors.Is(err, store.ErrLayerStackUndecided) { + t.Errorf("err = %v, want the read error", err) + } +} + +// A copy is flagged only when its group asks for layers it does not have. +func TestUnappliedGroupStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedGroups(t, st) + + for _, tc := range []struct { + group string + stack []string + want string + }{ + {"grp-luks", []string{"DRBD", "STORAGE"}, "DRBD,LUKS,STORAGE"}, + {"grp-luks", []string{"storage", "luks", "drbd"}, ""}, + {"grp-plain", []string{"DRBD", "STORAGE"}, ""}, + {"", []string{"DRBD", "STORAGE"}, ""}, + } { + got, err := store.UnappliedGroupStack(t.Context(), st, tc.group, tc.stack) + if err != nil || got != tc.want { + t.Errorf("UnappliedGroupStack(%q, %v) = %q, %v; want %q", tc.group, tc.stack, got, err, tc.want) + } + } +} + +func seedGroupedPreparedTarget(t *testing.T, st store.Store, group string, stack []string) *apiv1.Snapshot { + t.Helper() + + ctx := t.Context() + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{Name: "pvc-src", LayerStack: apiv1.DefaultLayerStack()}); err != nil { + t.Fatalf("seed source: %v", err) + } + + snap := &apiv1.Snapshot{ + Name: "snap-1", ResourceName: "pvc-src", + VolumeDefinitions: []apiv1.SnapshotVolumeDef{{VolumeNumber: 0, SizeKib: 1024}}, + } + + if err := st.ResourceDefinitions().Create(ctx, &apiv1.ResourceDefinition{ + Name: "pvc-dst", ResourceGroupName: group, LayerStack: stack, + }); err != nil { + t.Fatalf("seed target: %v", err) + } + + if err := st.VolumeDefinitions().Create(ctx, "pvc-dst", &apiv1.VolumeDefinition{VolumeNumber: 0, SizeKib: 1024}); err != nil { + t.Fatalf("seed target volume: %v", err) + } + + return snap +} + +// A prepared target whose stack its group has yet to decide is not judged +// prepared and nothing is written to it; a decided one of the source's stack is. +func TestPreparedRestoreTargetWaitsForAnUndecidedStack(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedGroups(t, st) + snap := seedGroupedPreparedTarget(t, st, "grp-luks", nil) + + target, _ := st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + + if _, err := store.PreparedRestoreTarget(t.Context(), st, &target, snap); !errors.Is(err, store.ErrLayerStackUndecided) { + t.Errorf("an undecided target = %v, want ErrLayerStackUndecided", err) + } + + if _, err := store.AdoptPreparedRestoreTarget(t.Context(), st, "pvc-dst", snap); !errors.Is(err, store.ErrLayerStackUndecided) { + t.Errorf("marking an undecided target = %v, want ErrLayerStackUndecided", err) + } + + target, _ = st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if target.Props[store.RestoreFromSnapshotProp] != "" || len(target.LayerStack) > 0 { + t.Errorf("the refused mark wrote %v / %v", target.Props, target.LayerStack) + } +} + +// The target is marked with the stack its group resolves to, read before the +// patch; a target moved to another group in between is answered as undecided +// rather than stamped with the old group's answer. +func TestRestoreMarkerResolvesTheStackItIsWrittenWith(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedGroups(t, st) + snap := seedGroupedPreparedTarget(t, st, "grp-plain", nil) + + moved := claimPatchHookStore{Store: st, hook: claimPatchHook{before: func() { + rd, _ := st.ResourceDefinitions().Get(context.Background(), "pvc-dst") + rd.ResourceGroupName = "grp-luks" + _ = st.ResourceDefinitions().Update(context.Background(), &rd) + }}} + + if _, err := store.AdoptPreparedRestoreTarget(t.Context(), moved, "pvc-dst", snap); !errors.Is(err, store.ErrLayerStackUndecided) { + t.Errorf("a target moved under the patch = %v, want ErrLayerStackUndecided", err) + } + + rd, _ := st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + rd.ResourceGroupName = "grp-plain" + _ = st.ResourceDefinitions().Update(t.Context(), &rd) + + if _, err := store.AdoptPreparedRestoreTarget(t.Context(), st, "pvc-dst", snap); err != nil { + t.Fatalf("marking a target under a group that sets no stack: %v", err) + } + + rd, _ = st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if !slices.Equal(rd.LayerStack, apiv1.DefaultLayerStack()) { + t.Errorf("marked target stack = %v, want %v", rd.LayerStack, apiv1.DefaultLayerStack()) + } +} + +// The target's stack is judged against the source's before the marker is +// stamped, and can be rewritten in between; the stamp checks it again against +// the state it applies to and refuses a stack the snapshot's bytes were not +// laid down under, leaving the target unmarked. +func TestRestoreMarkerRefusesAStackRewrittenUnderThePatch(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedGroups(t, st) + snap := seedGroupedPreparedTarget(t, st, "grp-plain", apiv1.DefaultLayerStack()) + + rewritten := claimPatchHookStore{Store: st, hook: claimPatchHook{before: func() { + rd, _ := st.ResourceDefinitions().Get(context.Background(), "pvc-dst") + rd.LayerStack = []string{"DRBD", "LUKS", "STORAGE"} + _ = st.ResourceDefinitions().Update(context.Background(), &rd) + }}} + + if _, err := store.AdoptPreparedRestoreTarget(t.Context(), rewritten, "pvc-dst", snap); !errors.Is(err, store.ErrRestoreTargetLayers) { + t.Errorf("a target whose stack was rewritten under the patch = %v, want ErrRestoreTargetLayers", err) + } + + rd, _ := st.ResourceDefinitions().Get(t.Context(), "pvc-dst") + if rd.Props[store.RestoreFromSnapshotProp] != "" { + t.Errorf("the refused stamp wrote the marker: %v", rd.Props) + } +} diff --git a/pkg/store/k8s/layer_stack_group_case_test.go b/pkg/store/k8s/layer_stack_group_case_test.go new file mode 100644 index 00000000..5bcfa89a --- /dev/null +++ b/pkg/store/k8s/layer_stack_group_case_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s_test + +import ( + "errors" + "slices" + "testing" + + apiv1 "github.com/cozystack/blockstor/pkg/api/v1" + "github.com/cozystack/blockstor/pkg/store" + "github.com/cozystack/blockstor/pkg/store/k8s" +) + +// The REST clone and restore doors resolve a definition's stack through the +// store. A definition naming its group in the operator's spelling resolves +// to the group's CR, stored under the folded name: undecided while the +// group's stack is not pinned onto it, and the pinned stack after, never +// the default stack the satellite would build for a group it cannot find. +func TestTheStackOfADefinitionNamingItsGroupInAnotherCase(t *testing.T) { + if fixture == nil { + t.Skip("envtest assets not installed; run `make setup-envtest` to enable") + } + + t.Cleanup(func() { wipeAll(t, fixture.client) }) + + ctx := t.Context() + st := k8s.New(fixture.client) + luks := []string{"DRBD", "LUKS", "STORAGE"} + + err := st.ResourceGroups().Create(ctx, &apiv1.ResourceGroup{ + Name: "LuksRG", SelectFilter: apiv1.AutoSelectFilter{LayerStack: luks}, + }) + if err != nil { + t.Fatalf("create group: %v", err) + } + + rd := apiv1.ResourceDefinition{Name: "pvc-case", ResourceGroupName: "LuksRG"} + + _, err = store.EffectiveLayerStack(ctx, st, &rd) + if !errors.Is(err, store.ErrLayerStackUndecided) { + t.Fatalf("before the pin: err = %v, want ErrLayerStackUndecided", err) + } + + rd.LayerStack = luks + + got, err := store.EffectiveLayerStack(ctx, st, &rd) + if err != nil || !slices.Equal(got, luks) { + t.Fatalf("after the pin: = %v, %v; want %v", got, err, luks) + } +} diff --git a/pkg/store/k8s/objectmeta_write_extended_bug_235_test.go b/pkg/store/k8s/objectmeta_write_extended_bug_235_test.go index 84e17eb1..4b06963d 100644 --- a/pkg/store/k8s/objectmeta_write_extended_bug_235_test.go +++ b/pkg/store/k8s/objectmeta_write_extended_bug_235_test.go @@ -96,6 +96,15 @@ var extendedObjectMetaWriteAllowList = []extendedObjectMetaWriteAllowListEntry{ file: "resource_groups.go", lineSubstring: "rg.Annotations = stripInternalAnnotations(rg.Annotations)", }, + // The volume-less clone builds a NEW definition from a value copy of + // its source and hands it to Create, so there is nothing stored to + // wipe. withoutUnappliedGroupStack returns a fresh map, which keeps the + // source's own map, possibly the in-memory store's, untouched. + { + dir: "pkg/rest", + file: "rd_clone.go", + lineSubstring: "clone.Annotations = withoutUnappliedGroupStack(src.Annotations)", + }, } type extendedObjectMetaWriteAllowListEntry struct { diff --git a/pkg/store/restore_target.go b/pkg/store/restore_target.go index d9e3f9dc..5e669db7 100644 --- a/pkg/store/restore_target.go +++ b/pkg/store/restore_target.go @@ -169,7 +169,16 @@ func PreparedRestoreTarget(ctx context.Context, st Store, cached *apiv1.Resource return false, fmt.Errorf("read the source %q of snapshot %q: %w", snap.ResourceName, snap.Name, err) } - have, want := EffectiveLayerStack(source.LayerStack), EffectiveLayerStack(rd.LayerStack) + have, err := EffectiveLayerStack(ctx, st, &source) + if err != nil { + return false, fmt.Errorf("the layer stack of the source %q: %w", source.Name, err) + } + + want, err := EffectiveLayerStack(ctx, st, rd) + if err != nil { + return false, fmt.Errorf("the layer stack of %q: %w", rd.Name, err) + } + if !sameLayerSet(have, want) { return false, fmt.Errorf("%w: %q has %s, the source %q has %s", ErrRestoreTargetLayers, rd.Name, strings.Join(want, ","), source.Name, strings.Join(have, ",")) @@ -178,22 +187,92 @@ func PreparedRestoreTarget(ctx context.Context, st Store, cached *apiv1.Resource return true, nil } -// EffectiveLayerStack resolves a stored stack the way the data plane does: the -// dispatcher hands the satellite the definition's own stack and nothing else, -// and the satellite brings up DRBD over STORAGE for an empty one. A parent -// group's stack is not part of it: it is copied onto a definition only when -// the definition is created through `rd c`. It returns a copy. +// ErrLayerStackUndecided reports a definition that leaves its layer stack to +// a resource group that sets one, which the controller has not yet written +// onto it. The satellite brings nothing up for it until then +// (checkLayerStackPinned), so the stack its volumes will have is not known: +// a clone or restore that took one now would be guessing. It clears on its +// own, once the controller pins the stack, so a retry is the answer. +var ErrLayerStackUndecided = errors.New("the resource definition's layer stack comes from its group and is not pinned yet") + +// EffectiveLayerStack is the stack the data plane brings rd's volumes up +// under. It returns a copy. // -// The control plane reads the group's stack for a definition stored without -// one (the quorum witness, the autoplace gate), so a restored or cloned -// definition left empty under a group with another stack is brought up as one -// shape and judged as another. Stamping this onto it keeps the two in step. -func EffectiveLayerStack(stack []string) []string { - if len(stack) == 0 { - return apiv1.DefaultLayerStack() +// - A definition's own stack is that stack. +// - An empty stack under no group, a group that is gone, or a group that +// sets no stack is the default stack, which is what the satellite builds. +// - An empty stack under a group that sets one is ErrLayerStackUndecided: +// the controller pins it once, to the group's stack when nothing is on +// disk yet and to the default stack when data is already there, and +// until then nothing is built. +// +// Clones and restores write this onto the definition that receives the +// source's bytes, because those bytes were laid down under it and a +// different stack over them either formats them away or reads them wrong. +// A group that cannot be read is an error, never a guess. +func EffectiveLayerStack(ctx context.Context, st Store, rd *apiv1.ResourceDefinition) ([]string, error) { + if len(rd.LayerStack) > 0 { + return slices.Clone(rd.LayerStack), nil + } + + groupStack, err := GroupLayerStack(ctx, st, rd.ResourceGroupName) + if err != nil { + return nil, err + } + + return layerStackUnder(rd.Name, rd.LayerStack, rd.ResourceGroupName, groupStack) +} + +// GroupLayerStack is the layer stack the named group sets, nil for no group, +// a group that is gone, or one that sets none. +func GroupLayerStack(ctx context.Context, st Store, rgName string) ([]string, error) { + if rgName == "" { + return nil, nil + } + + rg, err := st.ResourceGroups().Get(ctx, rgName) + if errors.Is(err, ErrNotFound) { + return nil, nil + } + + if err != nil { + return nil, fmt.Errorf("read resource group %q for its layer stack: %w", rgName, err) } - return append([]string(nil), stack...) + return rg.SelectFilter.LayerStack, nil +} + +// layerStackUnder is EffectiveLayerStack once the group's stack is known. +func layerStackUnder(rdName string, own []string, rgName string, groupStack []string) ([]string, error) { + if len(own) > 0 { + return slices.Clone(own), nil + } + + if len(groupStack) == 0 { + return apiv1.DefaultLayerStack(), nil + } + + return nil, fmt.Errorf("%w: %q in group %q", ErrLayerStackUndecided, rdName, rgName) +} + +// UnappliedGroupStack is the value of apiv1.UnappliedGroupLayerStackAnnotation +// for a definition that holds data under stack in the named group: the group's +// stack when the group sets one with other layers, "" otherwise. A clone or +// restore target carries its source's bytes, so its stack is the source's and +// not a choice; when that leaves it short of what its group asks for (a +// plaintext copy in a LUKS group), it is flagged the way the controller flags +// a definition it pinned over data. +func UnappliedGroupStack(ctx context.Context, st Store, rgName string, stack []string) (string, error) { + groupStack, err := GroupLayerStack(ctx, st, rgName) + if err != nil { + return "", err + } + + if len(groupStack) == 0 || sameLayerSet(stack, groupStack) { + return "", nil + } + + return strings.Join(groupStack, ","), nil } // sameLayerSet compares two stacks as sets. Order follows from the kinds in a @@ -244,9 +323,35 @@ func sameVolumes(vds []apiv1.VolumeDefinition, recorded []apiv1.SnapshotVolumeDe func stampRestoreMarker(ctx context.Context, st Store, rdName string, snap *apiv1.Snapshot) (bool, error) { marker := snap.ResourceName + ":" + snap.Name + // The stack is resolved inside the patch, against the state it applies + // to, so the group it is resolved under is read first; a definition that + // moved to another group in between is answered as undecided and retried. + target, err := st.ResourceDefinitions().Get(ctx, rdName) + if err != nil { + return false, fmt.Errorf("read %q before stamping the restore marker: %w", rdName, err) + } + + groupStack, err := GroupLayerStack(ctx, st, target.ResourceGroupName) + if err != nil { + return false, err + } + + // The target was judged to have the source's layers before this call, + // and its stack can be rewritten between that judgement and the patch; + // the marker goes on only over the stack the snapshot's bytes need. + source, err := st.ResourceDefinitions().Get(ctx, snap.ResourceName) + if err != nil { + return false, fmt.Errorf("read the source %q of snapshot %q: %w", snap.ResourceName, snap.Name, err) + } + + sourceStack, err := EffectiveLayerStack(ctx, st, &source) + if err != nil { + return false, fmt.Errorf("the layer stack of the source %q: %w", source.Name, err) + } + var wrote bool - err := st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, func(rd *apiv1.ResourceDefinition) error { + err = st.ResourceDefinitions().PatchResourceDefinitionSpec(ctx, rdName, func(rd *apiv1.ResourceDefinition) error { current := rd.Props[RestoreFromSnapshotProp] if current != "" && !strings.EqualFold(current, marker) { return fmt.Errorf("%w: %q carries %q", ErrRestoreTargetTaken, rdName, current) @@ -256,8 +361,22 @@ func stampRestoreMarker(ctx context.Context, st Store, rdName string, snap *apiv // run's, against the state the patch actually applied to. wrote = current == "" + if len(rd.LayerStack) == 0 && !strings.EqualFold(rd.ResourceGroupName, target.ResourceGroupName) { + return fmt.Errorf("%w: %q moved to group %q", ErrLayerStackUndecided, rdName, rd.ResourceGroupName) + } + + stack, err := layerStackUnder(rdName, rd.LayerStack, rd.ResourceGroupName, groupStack) + if err != nil { + return err + } + + if !sameLayerSet(sourceStack, stack) { + return fmt.Errorf("%w: %q has %s, the source %q has %s", ErrRestoreTargetLayers, + rdName, strings.Join(stack, ","), source.Name, strings.Join(sourceStack, ",")) + } + rd.Props = WithRestoreMarker(rd.Props, snap) - rd.LayerStack = EffectiveLayerStack(rd.LayerStack) + rd.LayerStack = stack return nil }) From 398b4fae8620d4337ec477fa1ef5d0a2fea98069 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:27:19 +0500 Subject: [PATCH 08/22] docs(layer-stack): say which stack a clone or restore takes Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/cli-parity-known-deltas.md | 2 +- docs/layer-stack.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/cli-parity-known-deltas.md b/docs/cli-parity-known-deltas.md index b2229fc2..92e86f2d 100644 --- a/docs/cli-parity-known-deltas.md +++ b/docs/cli-parity-known-deltas.md @@ -54,7 +54,7 @@ Row IDs match the command-catalogue indexes used by `cli-parity-refresh.sh` (see | 83 | `rg spawn-resources` on an over-committed RG (place_count > available nodes) | BEHAVIOR | permanent | Corner D1. Upstream LINSTOR fails `spawn-resources` SHORT when the RG's place_count exceeds the placeable node count: it returns `FAIL_NOT_ENOUGH_NODES` (ret_code 996, "Not enough available nodes") and places NOTHING. blockstor instead takes a DEFERRED, best-effort autoplace path: it spawns the RD + VDs, places as many diskful replicas as the topology allows (e.g. 3 of 7 on a 3-node cluster), and surfaces the shortfall as an INFO in the SUCCESS envelope (`resource definition spawned, autoplace deferred: : not enough candidate storage pools: placed N of M`, exit 0). The `RGRebalanceReconciler` (`internal/controller/rg_rebalance_controller.go`) then tops the replica count back up additively once more nodes appear. The over-commit `rg create` itself is ACCEPTED by both controllers (parity — never an early create-time error). Rationale: the deferred path lets the CSI external-provisioner retry loop converge on a created RD instead of hard-failing on a cluster that is one node away from satisfying the request, and is strictly additive (scale-down stays an explicit `r d`). Switching spawn back to the upstream 996 short-fail would be a deliberate API change, not a silent regression. Pinned by `pkg/rest/spawn_test.go::TestSpawnImpossiblePlacementReturnsActionableError` + `TestSpawnPartialFlagAllowsShortPlacement` (L1) and `tests/e2e/cli-matrix/rg-c-overcommit-spawn-defers.sh` (L6). | | 84 | `r c ` (tiebreaker→diskful promotion runs full SyncTarget, not skip-sync) | BEHAVIOR | permanent | Promoting a tiebreaker (diskless witness) to diskful runs a FULL SyncTarget on the promoted node instead of the upstream skip-sync. DELIBERATE: skip-sync would require BS to pre-stamp a matching day0 GI and let the fresh replica win the auto-primary election — but a diskful peer already holds data (`anyDiskfulPeerHasData == true`, `pkg/dispatcher/dispatcher.go`), so force-priming the fresh replica mints an UNRELATED Current UUID; the data-bearing peer then declines the handshake (`uuid_compare()=unrelated-data`, "Unrelated data, aborting!") and the pair wedges in mutual StandAlone that never auto-recovers (the respawn-StandAlone P0). BS therefore gates the auto-primary seed on `!anyDiskfulPeerHasData(peers)` and `!rdInitialized(rd)`: with a data-bearing peer present the promoted replica comes up Inconsistent and SyncTargets the real bytes off the peer. Data converges correctly; the only cost is a full resync of the volume — the safe trade vs. a StandAlone wedge. Upstream's skip-sync-on-promotion is not reproducible without reintroducing the wedge. Pinned by `tests/e2e/cli-matrix/r-c-over-tiebreaker-skip-sync.sh` (L6, asserts SyncTarget→UpToDate convergence + Bug 348 SyncSource shape) + the dispatcher auto-primary gate tests (`pkg/dispatcher/dispatcher_test.go`, respawn-StandAlone wedge regression). | | 85 | `r l` State column — bare `SyncSource`/`SyncTarget` literal (no `(NN%)` suffix) when OutOfSyncKib<=0 | WIRE_SHAPE | permanent | A replica actively resyncing renders `SyncTarget(NN%)` / `SyncSource(NN%)` WHILE there is data to copy. When the per-volume `OutOfSyncKib` is <= 0 (or the VD size is unknown), BS DELIBERATELY renders the BARE literal `SyncSource` / `SyncTarget` with NO `(NN%)` suffix: `withSyncPercent` (pkg/rest/resources.go, called from `annotateSyncProgress`) short-circuits on `outOfSyncKib<=0` rather than emit a misleading `(0%)`/`(100%)`. The drbd replication-state literal can be observed for a brief window before/after the peer-device OutOfSyncKib counter carries a non-zero value, so a bare Sync* token is a real and intended shape. The REGRESSION guarded against is the opposite — a TERMINAL `UpToDate` must never carry a `(NN%)` suffix (Bug A). So the contract is: a Sync* token may render bare OR with `(NN%)`, but a terminal disk_state never carries a percent. Pinned by `tests/e2e/cli-matrix/r-l-conns-shapes.sh` sub-test D (L6) + `annotateSyncProgress`/`withSyncPercent` unit coverage (pkg/rest, Bug 348 + Bug A/B). | -| 86 | `rd clone --external-name` / `--volume-passphrase` / a `--layer-list` that changes the source's layer set | MISSING_FEATURE | permanent | `external_name` gives the clone an identity of its own upstream and `volume_passphrases` carries the LUKS keys for its volumes; blockstor names a clone by `name` and takes its keys from the cluster passphrase, so honouring neither is possible and both are refused with an explicit 501 rather than accepted and dropped. Every refusal on this endpoint, the 501 included, goes out in the shape the caller decodes: the CloneStarted object for python-linstor, which names itself `PythonLinstor/…` and reads the clone answer as that object whatever the status, and an ApiCallRc array for every other client, since golinstor decodes a non-2xx body as nothing else (linstor-csi's 404s reach it bare either way) — a clone that reports success under a different name, or with keys the caller does not hold, is the failure mode `src_snap_name` is already refused for. `layer_list` IS honoured, with one refusal: on a source that has volumes the requested stack must be the source's, compared as a set (order is the stack's own and case folds). The clone data plane restores the source's bytes and brings the layer stack up over them, in that order, and every layer's bring-up writes to the device — LUKS formats a device carrying no header, DRBD stamps `meta-disk internal` metadata with `create-md --force` — so a layer added here lands across the bytes just restored and the clone reports COMPLETE over them, while a layer dropped leaves the target reading data the missing layer wrote. A source with no recorded stack means the upstream default `[DRBD, STORAGE]`, which is what linstor-csi sends, so the CSI clone path is unaffected. Upstream clones the layer list with the volume, re-encrypting LUKS under the cluster key. A volume-less source has no bytes to lose and keeps taking any stack. A clone resumed over a leftover also keeps that leftover's shape: a retry naming a different `resource_group` or stack is refused rather than answered 201 with the request's shape dropped, and a leftover internal snapshot that has fallen behind the source (a volume added, or resized) is refused rather than cloned from. Pinned by `pkg/rest/rd_clone_golinstor_shape_test.go` + `pkg/rest/rd_clone_client_envelope_test.go` + `pkg/rest/rd_clone_layer_stack_test.go` + `pkg/rest/rd_clone_resume_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/rd_clone_server_owned_props_test.go` + `pkg/rest/rd_modify_server_owned_props_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). `rd modify` and `rd set-property` / `delete-property` refuse the same server-owned props, with 400 and an error respectively, since a `delete_namespaces` of `Blockstor` mid-restore took the adoption mark with it, and `rd create` refuses a definition carrying one with 400. Prop edits on a clone (`override_props`, `delete_props`, `delete_namespaces`) that would rewrite or remove a prop blockstor sets on the clone itself, the restore marker, the record of the snapshot's volumes, the rollback mark or the adoption mark, are refused with 400 before anything is written, rather than applied after the marker and leaving the satellite to bring the volumes up blank; linstor-csi's own `delete_props` edit touches none of them. | +| 86 | `rd clone --external-name` / `--volume-passphrase` / a `--layer-list` that changes the source's layer set | MISSING_FEATURE | permanent | `external_name` gives the clone an identity of its own upstream and `volume_passphrases` carries the LUKS keys for its volumes; blockstor names a clone by `name` and takes its keys from the cluster passphrase, so honouring neither is possible and both are refused with an explicit 501 rather than accepted and dropped. Every refusal on this endpoint, the 501 included, goes out in the shape the caller decodes: the CloneStarted object for python-linstor, which names itself `PythonLinstor/…` and reads the clone answer as that object whatever the status, and an ApiCallRc array for every other client, since golinstor decodes a non-2xx body as nothing else (linstor-csi's 404s reach it bare either way) — a clone that reports success under a different name, or with keys the caller does not hold, is the failure mode `src_snap_name` is already refused for. `layer_list` IS honoured, with one refusal: on a source that has volumes the requested stack must be the source's, compared as a set (order is the stack's own and case folds). The clone data plane restores the source's bytes and brings the layer stack up over them, in that order, and every layer's bring-up writes to the device — LUKS formats a device carrying no header, DRBD stamps `meta-disk internal` metadata with `create-md --force` — so a layer added here lands across the bytes just restored and the clone reports COMPLETE over them, while a layer dropped leaves the target reading data the missing layer wrote. The source's stack is the one its volumes are laid down under: its own, or the upstream default `[DRBD, STORAGE]` when neither it nor its group sets one, which is what linstor-csi sends, so the CSI clone path is unaffected. A source that leaves its stack to a group that sets one has that stack decided once by the controller (`docs/layer-stack.md`), and a clone or restore of it is answered 503 until then rather than guessing; the copy takes the decided stack, and one short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation, like a definition the controller pinned over data. Upstream clones the layer list with the volume, re-encrypting LUKS under the cluster key. A volume-less source has no bytes to lose and keeps taking any stack; one that never set its own passes none on, so the shell's own group decides it the same way. A clone resumed over a leftover also keeps that leftover's shape: a retry naming a different `resource_group` or stack is refused rather than answered 201 with the request's shape dropped, and a leftover internal snapshot that has fallen behind the source (a volume added, or resized) is refused rather than cloned from. Pinned by `pkg/rest/rd_clone_golinstor_shape_test.go` + `pkg/rest/rd_clone_client_envelope_test.go` + `pkg/rest/rd_clone_layer_stack_test.go` + `pkg/rest/rd_clone_resume_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/rd_clone_server_owned_props_test.go` + `pkg/rest/restore_effective_layers_test.go` + `pkg/store/effective_layer_stack_test.go` + `pkg/rest/rd_modify_server_owned_props_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). `rd modify` and `rd set-property` / `delete-property` refuse the same server-owned props, with 400 and an error respectively, since a `delete_namespaces` of `Blockstor` mid-restore took the adoption mark with it, and `rd create` refuses a definition carrying one with 400. Prop edits on a clone (`override_props`, `delete_props`, `delete_namespaces`) that would rewrite or remove a prop blockstor sets on the clone itself, the restore marker, the record of the snapshot's volumes, the rollback mark or the adoption mark, are refused with 400 before anything is written, rather than applied after the marker and leaving the satellite to bring the volumes up blank; linstor-csi's own `delete_props` edit touches none of them. | | 87 | `rd clone` / `s resource restore` retry over an unfinished target | BEHAVIOR | permanent | This is the data path: a clone of a source with volumes, and every restore. A clone of a volume-less source is a plain copy with no marker, so a repeat of it stays the already-exists refusal upstream gives. Both endpoints stamp `BlockstorRestoreFromSnapshot` with the target definition and hydrate its volumes and place its replicas afterwards, so a target carrying the marker says the operation STARTED, not that it finished. A repeat under the same name RESUMES it — every step tolerates an object a previous attempt already created — and says so (`… completed on retry`); upstream has no equivalent notion, a repeat there is a plain already-exists failure. This is what makes CSI's CreateVolume idempotent: external-provisioner has no other way to make progress after a partial failure, and answering success on the marker alone would leave CSI seeing a ready volume nothing ever finishes. A definition under that name that carries no marker is taken when it is in the state LINSTOR's own sequence leaves (`rd c`, `s vd restore`, then `s rsc restore`): its volumes exactly the snapshot's and no live replica. linstor-csi restores every volume that way, creating the definition itself first, and the marker is stamped on it at that point. What stays a refusal, each before anything is written to the definition: any other definition under that name this operation did not create and that is not prepared, meaning its volumes are not the snapshot's or it holds a live replica (409); a prepared definition whose layer stack is not the snapshot source's, compared as a set, since the restore would put the source's bytes under layers it never had or without layers it had (409); a prepared definition, or a leftover still to be finished or one the request places replicas into, whose every replica is still being deleted, which placing over would race (409; a bare REST restore's replay over a finished leftover still answers 201 while its snapshot exists, since it places none); a prepared definition or a leftover parented to a resource group that no longer exists (409, or 500 asking for a retry when the group cannot be read); this operation's own leftover carrying the DELETE flag, since finishing it races the tear-down reaping what it writes (409); a leftover whose earlier attempt's rollback gave up, which may hold less than the operation intended, with the way out named (409); a leftover holding a volume the snapshot never recorded beside one it did (409); and a leftover whose creating attempt has started rolling it back (409). Placing a replica, after the definition and its volumes are written, refuses an operator's diskless replica already on a node the request places onto, which holds no copy of the data and is not counted as placed: a 409 on a retry over this operation's own leftover, and on a fresh restore a failed placement, which rolls the new definition back and answers 500, neither with a FAIL_EXISTS band. The controller's tie-breaker witness there is promoted to the replica the request asked for instead, as autoplace promotes one, and its volumes are restored from the snapshot like any other replica of the target; a witness no storage pool can be found for is left as it is and refused the same way, naming `linstor resource create --storage-pool ` to give it one. Once the marker is on a prepared definition it stays: a concurrent retry that finds it resumes and places the replica, and the request that wrote it meets that replica as one already placed, never as a reason to take the marker back off. The window this leaves is a replica placed by hand between the judgement and the marker, which is then judged as part of the restore; on a target prepared for exactly this restore nothing else places one in that moment. A replica the operation would place that is still being deleted on its node fails the request rather than counting as placed (a 409 over a leftover this request adopted, and a 500 with the rollback's answer over one it created), with no FAIL_EXISTS_RSC band on either. Two restores of different snapshots into one prepared definition do not both take it: the second finds the first's marker and is refused. The CLI door settles the parent group, and the nodes and pools the replicas go to, before it writes the definition or the marker, so a request refused for its own shape, or over a prepared definition whose group is gone, leaves that definition unmarked; and a CLI placement that fails after the marker is finished by running the same command again, unless a replica already landed: the re-run then judges it finished, so the failure names the replicas that did not land and the `blockstor resource create` that places them. The CLI judges its own leftover with the same assessment and gate order the REST door uses (a tear-down, then a volume the restore would not have written, then a parent group that is gone, then a rollback that gave up): a leftover with the snapshot's volumes and one replica with a disk is finished and left alone rather than re-placed on a node the operator emptied, and only one with no such replica gets its placement finished, under the same adoption mark the REST door takes, so a rollback by the attempt that created it yields. A definition already under the name is judged before anything about the source is planned, so a finished restore is left alone even once the source can no longer be planned from. Replicas are counted past the cache on both doors, and a rollback still in progress is refused as one still running rather than as one that gave up. A clone resume whose internal snapshot is gone is refused once the leftover holds a volume with a replica that has a disk, since finishing it from a new snapshot would join two moments of the source; a leftover with no volume or no replica is finished from the new snapshot like a fresh clone, unless its volumes do not fit the source as it is now (one smaller than the source's, or one the source no longer has), which is refused with `delete and clone again` before any snapshot is taken. The volumes and replicas of a prepared definition, and a colliding replica, are read from the API server, not a cache, and so is the definition the request judges right after stamping its marker: linstor-csi's volume restore and the resource restore after it can be served by different replicas of the server. A finished clone or restore is judged by its volumes and, where the operation placed replicas, by holding one with a disk, never by where they are now: a replay after a node was evacuated or the target scaled down answers success and leaves placement alone. A finished clone is a copy of a point-in-time and its replay answers the same way whatever happens to the source afterwards — including a `vd set-size`. Pinned by `pkg/rest/rd_clone_idempotency_test.go` + `pkg/rest/rd_clone_replay_test.go` + `pkg/rest/rd_clone_leftover_shape_test.go` + `pkg/rest/snapshot_restore_idempotency_test.go` + `pkg/rest/snapshot_restore_replay_test.go` + `pkg/rest/snapshot_restore_csi_flow_test.go` + `pkg/rest/snapshot_restore_prepared_target_test.go` + `pkg/rest/snapshot_restore_prepared_cache_test.go` + `pkg/rest/snapshot_restore_resume_gates_test.go` + `pkg/rest/leftover_gates_test.go` + `pkg/rest/rollback_handshake_failclosed_test.go` + `pkg/store/restore_target_test.go` + `internal/cli/snapshot_restore_prepared_test.go` (L1) and `tests/e2e/cli-matrix/rd-clone-retry-semantics.sh` (L6, stand-only: `make e2e-list` covers `tests/e2e/*.sh` and no CI lane runs the cli-matrix cells) + `tests/operator-harness/replay/rd-clone-retry-semantics.yaml` (L7). Every marker is written with `Blockstor/RestoreVolumes`, the volumes the snapshot held, so a leftover whose snapshot is gone (an operator deleted it) is still judged against them: a clone that restored only some of its volumes is unfinished, not complete. A leftover from before the record existed carries none and is judged on its own volumes, as before. A CLI rollback that could not delete takes its in-progress mark back off so the next run finishes the definition, and a CLI re-run over a finished restore says so on stdout. The CLI's rollback of a definition it created holds the same handshake as REST before deleting: a definition another request adopted is left to it, and one whose mark could not be written or whose adoption could not be read is not deleted. A rollback after a failed clone or restore also leaves a definition that already reads as finished, holding every volume and, where the operation places replicas, one replica with a disk: the clone status poll answers such a definition complete without writing anything, which can happen while the attempt that created it is still stamping its last replica, so deleting it afterwards took a volume the driver had been told exists. A replay takes the adoption mark before its 201, so the rollback yields to it like to any adopter, whatever the request placed. The rollback over a group deleted under the operation is the exception: it removes a finished definition too, since one parented to a group that is gone is what it exists to remove, so a status poll that answered complete a moment before it starts is followed by the definition going away; a POST replay's mark makes it yield. That attempt fails, and its retry answers for what it left; a restore that asked for replicas on named nodes names the ones it never placed, with the `linstor resource create` that places them, since the retry judges the definition finished and places nothing. A clone's prop edits (`override_props`, `delete_props`, `delete_namespaces`) land with the definition when it is created, so no leftover carries a prop the caller asked to remove: linstor-csi removes `Aux/csi-provisioning-completed-by` on every clone and takes a definition carrying it, with volumes of the right size, as one already provisioned. The clone status poll asks a finished target what the POST's replay asks before its 201: one carrying the DELETE flag, parented to a group that is gone, or carrying a rollback in progress or one that gave up is answered 404, which sends the driver back to the POST that refuses it, and a group that cannot be read is a 500. A replay of a finished clone is answered before the source is read, so it survives the source being deleted or losing its volumes. A retry after the snapshot was deleted, the cleanup that follows a finished restore, is judged against the volumes the definition recorded when it was restored: a finished one answers 201 (the CLI exits 0) without reading the snapshot or the source, an unfinished one is refused with `delete and restore it from another snapshot`, and a target restored before the record existed keeps the 404 it got. A restored or cloned definition is stamped with the source's data-plane layer stack (its own stored stack, or DRBD over STORAGE when it has none): the dispatcher reads only the definition's stack, so a target group with another stack never changed how the bytes are brought up, but the control plane reads the group's stack for a definition stored without one, and the stamp keeps the two in step. A REST replay of a finished restore checks its node list only while the snapshot exists; once it is gone the replay is judged by the target alone. A bare restore counts as finished without a replica only while its snapshot exists: once the snapshot is gone, a target with no replica holds its data nowhere and is refused with `delete and restore it from another snapshot`, as the CLI refuses it. A clone that names no layer stack takes the source's and needs the cluster passphrase when that stack carries LUKS, as one that names it does; the check runs after the replay of a finished clone, which writes no new LUKS volume. A target already under the name is judged, hydrated and placed under the name it is stored with, whatever case the request spells it in: its replicas are selected by that name exactly, and judged under another spelling a finished clone or restore read as having none; a read of that name that fails is the 500 below. A target that cannot be read is answered 500 asking for a retry, not taken as absent: the create that followed collided and was adopted past every judgement of a leftover. The CLI refuses a snapshot that recorded no volumes before it writes or resumes anything, as the REST door does. A rollback that leaves a definition whole and cannot take its in-progress mark back off says so and names `linstor resource-definition set-property BlockstorRollbackAbandoned` with no value (python-linstor's delete; the blockstor CLI also has `delete-property`), which `rd modify` (`delete_props`) and the CLI accept only while the mark sits on a definition the rollback left whole (`in-progress`, or `snapshots` / `read-snapshots`, where it stopped before touching anything): one naming a later step sits on a definition the rollback took partly apart, and stays. It is one of three server-owned props an operator may delete by name, on both doors. The second is `Blockstor/RestoreAdopted`, which a claim whose patch landed and whose answer and read-back both failed leaves with no request answering for it; every later rollback yields on it, so where the clone or restore can never finish only the operator takes it off. The third is `BlockstorRestoreFromSnapshot` once the definition holds every volume and a diskful replica whose satellite reports every volume UpToDate: the marker keeps pinning placement to the snapshot's backend and nodes after the data is restored, and deleting it is the way to lift that pin, as it was on main; deleting it from an unfinished restore is refused, since a replica placed without it comes up blank. Setting any of them stays refused, and an empty `override_props` value is read as the delete. The refusal over an in-progress mark names the clear only once the rollback has had its time; a clear while it still runs stops that rollback before it deletes anything, since the rollback checks its own mark is still there before its first delete. A CLI rollback that leaves the target in place names the `blockstor snapshot resource restore` that finishes it, including for `rd clone`, whose own re-run is refused over a target that exists. | | 88 | `rg vg c` / `rg vg m` / `rg c` with volume-group `flags` | MISSING_FEATURE | permanent | Upstream's one volume-group flag is `GROSS_SIZE`, which sizes a spawned volume gross. Nothing in blockstor sizes a spawned volume gross, so both volume-group doors answer 501 to a body that sets a flag and write nothing, the props half included, rather than a 200 for a request that changes nothing. The volume-group create door and `rg c`, whose body can carry volume groups of its own, refuse any `flags`, so a flag no longer reaches the store; on main it was stored, listed by `rg vg l`, and did nothing. The modify door honours removals, spelled `-` as golinstor's `VolumeGroupModify` documents, since that is the only way to clear a flag a volume group created before this already carries; a body that mixes a removal with a set is refused whole. Pinned by `pkg/rest/vg_modify_golinstor_test.go`. | | 89 | `delete_namespaces` on every props-modify door | BEHAVIOR | permanent | Recorded so it is not reopened. As upstream's `PropsContainer.removeNamespace` does, a namespace removes the keys below it and leaves a key spelled exactly as the namespace: `DrbdOptions` takes `DrbdOptions/Net/protocol` and keeps `DrbdOptions`. Trailing slashes name the same namespace, however many there are. Against main this both widens and narrows the delete. Wider: a trailing-slash entry deleted nothing on most doors (the node-connection door trimmed one slash), and a multi-slash entry such as `Aux//` deleted nothing on any door; now both delete the keys below `Aux`. Narrower: the volume-definition, per-resource volume and node-connection doors deleted a key spelled exactly like the namespace, and now keep it. One spelling still differs from upstream: an entry of `/` alone, which upstream reads as the root namespace and so clears every prop, names nothing here and deletes nothing, the safe side of the difference. Every door and every guard in front of one read it through the same helper (`store.NamespaceCovers`), so a guard in front of a door refuses every spelling the delete honours. | diff --git a/docs/layer-stack.md b/docs/layer-stack.md index 8941df8e..eaf404bd 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -34,6 +34,8 @@ An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The RE - An RD that holds no data yet gets the group's stack. - An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. The group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. +Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack only when the source set one by choice, and otherwise is left for the controller to pin from its own group. + Changing a group's stack later never reshapes RDs that already carry one, as in upstream LINSTOR, where the stack is decided when a resource is created. As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. On a fresh thick LV that reuses extents of an earlier volume the signature can be stale; clear it with `wipefs -a ` once you have checked it holds nothing you need. From 6f448fd41ed6b3c510815803d6eaef95e5554583 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:34:58 +0500 Subject: [PATCH 09/22] fix(controller): read a group's layer stack as a set when pinning The pin compared the stack it applied with the group's in order and case, while clone and restore compared them as sets. A group spelling its stack in another order or case flagged every definition holding data as short of it. Both now share one comparison. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- .../resourcedefinition_layer_stack.go | 2 +- .../resourcedefinition_layer_stack_test.go | 17 +++++++++++++ pkg/api/v1/layer_stack.go | 18 ++++++++++++++ pkg/store/restore_target.go | 24 +++---------------- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index 786040f2..1783e487 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -90,7 +90,7 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl base := rd.DeepCopy() rd.Spec.LayerStack = slices.Clone(stack) - unapplied := !slices.Equal(stack, groupStack) + unapplied := !apiv1.SameLayerSet(stack, groupStack) if unapplied { if rd.Annotations == nil { rd.Annotations = map[string]string{} diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go index 11fa54ea..0c653d0c 100644 --- a/internal/controller/resourcedefinition_layer_stack_test.go +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -237,6 +237,23 @@ func TestPinLayerStackDataUnderTheGroupsOwnStackIsNoConflict(t *testing.T) { } } +// The group's stack is a set: a group spelling it in another order or case +// asks for the same layers, and the data under them is no conflict. +func TestPinLayerStackReadsTheGroupsStackAsASet(t *testing.T) { + t.Parallel() + + for _, groupStack := range [][]string{{"drbd", "storage"}, {"STORAGE", "DRBD"}} { + got := reconcilePin(t, pinClient(t, pinGroup(groupStack...), pinRD("rg-pin"), withMetadata(pinReplica("pvc-pin", "n1")))) + if got.err != nil { + t.Fatalf("reconcile under %v: %v", groupStack, got.err) + } + + if _, ok := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; ok || len(got.events) != 0 { + t.Errorf("group %v: annotations %v, events %v, want neither", groupStack, got.rd.Annotations, got.events) + } + } +} + // Data on another RD's replica says nothing about this one. func TestPinLayerStackLooksOnlyAtItsOwnReplicas(t *testing.T) { t.Parallel() diff --git a/pkg/api/v1/layer_stack.go b/pkg/api/v1/layer_stack.go index bf83ea26..2e2d603c 100644 --- a/pkg/api/v1/layer_stack.go +++ b/pkg/api/v1/layer_stack.go @@ -52,6 +52,24 @@ func LayerInStack(stack []string, kind string) bool { return false } +// SameLayerSet compares two stacks as sets. Order follows from the kinds in a +// stack and LINSTOR folds name case, so neither is a difference. +func SameLayerSet(have, want []string) bool { + for _, layer := range have { + if !LayerInStack(want, layer) { + return false + } + } + + for _, layer := range want { + if !LayerInStack(have, layer) { + return false + } + } + + return true +} + // ContainsReplicationLayer reports whether stack contains a // replication-capable layer. Today that's DRBD only — DRBD-9 is the // sole layer that ships block-level inter-node replication and the diff --git a/pkg/store/restore_target.go b/pkg/store/restore_target.go index 5e669db7..0c265382 100644 --- a/pkg/store/restore_target.go +++ b/pkg/store/restore_target.go @@ -179,7 +179,7 @@ func PreparedRestoreTarget(ctx context.Context, st Store, cached *apiv1.Resource return false, fmt.Errorf("the layer stack of %q: %w", rd.Name, err) } - if !sameLayerSet(have, want) { + if !apiv1.SameLayerSet(have, want) { return false, fmt.Errorf("%w: %q has %s, the source %q has %s", ErrRestoreTargetLayers, rd.Name, strings.Join(want, ","), source.Name, strings.Join(have, ",")) } @@ -268,31 +268,13 @@ func UnappliedGroupStack(ctx context.Context, st Store, rgName string, stack []s return "", err } - if len(groupStack) == 0 || sameLayerSet(stack, groupStack) { + if len(groupStack) == 0 || apiv1.SameLayerSet(stack, groupStack) { return "", nil } return strings.Join(groupStack, ","), nil } -// sameLayerSet compares two stacks as sets. Order follows from the kinds in a -// stack and LINSTOR folds name case, so neither is a difference. -func sameLayerSet(have, want []string) bool { - for _, layer := range have { - if !apiv1.LayerInStack(want, layer) { - return false - } - } - - for _, layer := range want { - if !apiv1.LayerInStack(have, layer) { - return false - } - } - - return true -} - // sameVolumes reports whether a definition's volumes are exactly the ones the // snapshot recorded: the same numbers at the same sizes, nothing more. func sameVolumes(vds []apiv1.VolumeDefinition, recorded []apiv1.SnapshotVolumeDef) bool { @@ -370,7 +352,7 @@ func stampRestoreMarker(ctx context.Context, st Store, rdName string, snap *apiv return err } - if !sameLayerSet(sourceStack, stack) { + if !apiv1.SameLayerSet(sourceStack, stack) { return fmt.Errorf("%w: %q has %s, the source %q has %s", ErrRestoreTargetLayers, rdName, strings.Join(stack, ","), source.Name, strings.Join(sourceStack, ",")) } From 529a7a5e341e0f4468ae19ed739cb56b6d4d1a69 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:34:58 +0500 Subject: [PATCH 10/22] fix(stand): let the controller record events through events.k8s.io The recorder writes events.k8s.io events, which the stand's controller role did not grant, so the layer-stack warning was dropped. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- stand/blockstor-deploy.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stand/blockstor-deploy.yaml b/stand/blockstor-deploy.yaml index 7b762e87..82747aaf 100644 --- a/stand/blockstor-deploy.yaml +++ b/stand/blockstor-deploy.yaml @@ -32,6 +32,9 @@ rules: - apiGroups: [""] resources: [events] verbs: [create, patch] + - apiGroups: [events.k8s.io] + resources: [events] + verbs: [create, patch] - apiGroups: [""] resources: [nodes] verbs: [get, list, watch] From afc11e67fab739fbe7ca9c351da29bb6ec169dd7 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:50:13 +0500 Subject: [PATCH 11/22] fix(layer-stack): flag only a definition short of its group's layers The unapplied-group-layer-stack annotation was set on any difference from the group's stack, so an encrypted copy in a plaintext group was flagged as if it were the plaintext volume the annotation exists to find. It is now set only when a layer the group asks for is missing. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- .../controller/resourcedefinition_layer_stack.go | 2 +- .../resourcedefinition_layer_stack_test.go | 15 +++++++++++++++ pkg/api/v1/annotations.go | 12 ++++++------ pkg/api/v1/layer_stack.go | 14 ++++++++++++++ pkg/store/effective_layer_stack_test.go | 3 +++ pkg/store/restore_target.go | 4 ++-- 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index eaf404bd..eab7ddfc 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -40,7 +40,7 @@ Changing a group's stack later never reshapes RDs that already carry one, as in As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. On a fresh thick LV that reuses extents of an earlier volume the signature can be stale; clear it with `wipefs -a ` once you have checked it holds nothing you need. -Find the RDs that kept the default stack: +Find the RDs short of a layer their group asks for: ```bash kubectl get resourcedefinitions.blockstor.cozystack.io \ diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index 1783e487..bcdb233d 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -90,7 +90,7 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl base := rd.DeepCopy() rd.Spec.LayerStack = slices.Clone(stack) - unapplied := !apiv1.SameLayerSet(stack, groupStack) + unapplied := apiv1.LacksLayerOf(stack, groupStack) if unapplied { if rd.Annotations == nil { rd.Annotations = map[string]string{} diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go index 0c653d0c..f16b48c1 100644 --- a/internal/controller/resourcedefinition_layer_stack_test.go +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -254,6 +254,21 @@ func TestPinLayerStackReadsTheGroupsStackAsASet(t *testing.T) { } } +// Data under more layers than the group asks for is not short of it, and is +// not flagged. +func TestPinLayerStackFlagsOnlyAStackShortOfTheGroup(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, pinGroup("STORAGE"), pinRD("rg-pin"), withMetadata(pinReplica("pvc-pin", "n1")))) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + if _, ok := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; ok || len(got.events) != 0 { + t.Errorf("annotations %v, events %v, want neither", got.rd.Annotations, got.events) + } +} + // Data on another RD's replica says nothing about this one. func TestPinLayerStackLooksOnlyAtItsOwnReplicas(t *testing.T) { t.Parallel() diff --git a/pkg/api/v1/annotations.go b/pkg/api/v1/annotations.go index 3431f525..4fe3181f 100644 --- a/pkg/api/v1/annotations.go +++ b/pkg/api/v1/annotations.go @@ -142,10 +142,10 @@ const PeerChangedAnnotation = "blockstor.io/peer-changed" const RDSpawnShortfallAnnotation = "blockstor.io/spawn-shortfall" // UnappliedGroupLayerStackAnnotation is stamped on a ResourceDefinition -// whose layer stack was left to its resource group but which already -// held data under the default stack when the controller came to fix -// it. The controller pins the stack the satellite actually applied and -// records here, comma-separated, the group's stack that was not, so an -// operator can find every volume that is not what its group promises -// (for example a plaintext volume in a LUKS group) and migrate it. +// whose data was written under a stack missing a layer its resource group +// asks for: one the controller pinned over data it found under the default +// stack, or a clone or restore of such bytes. It records, comma-separated, +// the group's stack, so an operator can find every volume that is short of +// what its group promises (for example a plaintext volume in a LUKS group) +// and migrate it. const UnappliedGroupLayerStackAnnotation = "blockstor.io/unapplied-group-layer-stack" diff --git a/pkg/api/v1/layer_stack.go b/pkg/api/v1/layer_stack.go index 2e2d603c..61f225ed 100644 --- a/pkg/api/v1/layer_stack.go +++ b/pkg/api/v1/layer_stack.go @@ -52,6 +52,20 @@ func LayerInStack(stack []string, kind string) bool { return false } +// LacksLayerOf reports whether stack is missing a layer group names, read as +// sets. A stack with layers of its own beyond the group's is not short of it: +// an encrypted volume in a plaintext group is not what an operator looking for +// a missing layer is after. +func LacksLayerOf(stack, group []string) bool { + for _, layer := range group { + if !LayerInStack(stack, layer) { + return true + } + } + + return false +} + // SameLayerSet compares two stacks as sets. Order follows from the kinds in a // stack and LINSTOR folds name case, so neither is a difference. func SameLayerSet(have, want []string) bool { diff --git a/pkg/store/effective_layer_stack_test.go b/pkg/store/effective_layer_stack_test.go index 8f58ac72..61946fdd 100644 --- a/pkg/store/effective_layer_stack_test.go +++ b/pkg/store/effective_layer_stack_test.go @@ -36,6 +36,7 @@ func seedGroups(t *testing.T, st store.Store) { for _, rg := range []apiv1.ResourceGroup{ {Name: "grp-luks", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "LUKS", "STORAGE"}}}, {Name: "grp-plain"}, + {Name: "grp-drbd", SelectFilter: apiv1.AutoSelectFilter{LayerStack: []string{"DRBD", "STORAGE"}}}, } { if err := st.ResourceGroups().Create(t.Context(), &rg); err != nil { t.Fatalf("seed group %q: %v", rg.Name, err) @@ -111,6 +112,8 @@ func TestUnappliedGroupStack(t *testing.T) { {"grp-luks", []string{"DRBD", "STORAGE"}, "DRBD,LUKS,STORAGE"}, {"grp-luks", []string{"storage", "luks", "drbd"}, ""}, {"grp-plain", []string{"DRBD", "STORAGE"}, ""}, + // More layers than the group asks for is not short of it. + {"grp-drbd", []string{"DRBD", "LUKS", "STORAGE"}, ""}, {"", []string{"DRBD", "STORAGE"}, ""}, } { got, err := store.UnappliedGroupStack(t.Context(), st, tc.group, tc.stack) diff --git a/pkg/store/restore_target.go b/pkg/store/restore_target.go index 0c265382..374ac34a 100644 --- a/pkg/store/restore_target.go +++ b/pkg/store/restore_target.go @@ -257,7 +257,7 @@ func layerStackUnder(rdName string, own []string, rgName string, groupStack []st // UnappliedGroupStack is the value of apiv1.UnappliedGroupLayerStackAnnotation // for a definition that holds data under stack in the named group: the group's -// stack when the group sets one with other layers, "" otherwise. A clone or +// stack when stack is missing one of its layers, "" otherwise. A clone or // restore target carries its source's bytes, so its stack is the source's and // not a choice; when that leaves it short of what its group asks for (a // plaintext copy in a LUKS group), it is flagged the way the controller flags @@ -268,7 +268,7 @@ func UnappliedGroupStack(ctx context.Context, st Store, rgName string, stack []s return "", err } - if len(groupStack) == 0 || apiv1.SameLayerSet(stack, groupStack) { + if !apiv1.LacksLayerOf(stack, groupStack) { return "", nil } From 4914b3baa591e560e9b856f6b550547364021782 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:50:13 +0500 Subject: [PATCH 12/22] fix(linstor-migrate): read a definition's stack off its diskful replicas LINSTOR builds a diskless replica as DRBD over STORAGE whatever the definition's stack, so counting it made the replicas of every LUKS definition with a tiebreaker look divergent and left its stack unset. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/linstormigrate/convert.go | 17 ++++++++++++++- pkg/linstormigrate/layer_stack_test.go | 29 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/pkg/linstormigrate/convert.go b/pkg/linstormigrate/convert.go index d3558f44..3e24060e 100644 --- a/pkg/linstormigrate/convert.go +++ b/pkg/linstormigrate/convert.go @@ -400,11 +400,22 @@ func (c *converter) buildLayerInstanceIndexes() { // then each base-suffix child down to STORAGE. The RD row's own // layer_stack is only what was asked for at creation, and is often // empty, LINSTOR having resolved the stack per resource; this chain is -// what is on disk. +// what is on disk. A diskless replica is left out: LINSTOR builds it as +// DRBD over STORAGE whatever the definition's stack, so it says nothing +// about how the data was written. func (c *converter) indexDiskStacks() { rows := c.dump.LayerResourceIDs children := map[int32]LayerResourceIDRow{} + diskless := map[replicaKey]bool{} + + for i := range c.dump.Resources { + res := &c.dump.Resources[i] + if res.SnapshotName == "" && res.ResourceFlags&resourceFlagDiskless != 0 { + diskless[replicaKey{node: res.NodeName, rd: res.ResourceName}] = true + } + } + for i := range rows { if rows[i].LayerResourceParentID != nil && rows[i].LayerResourceSuffix == "" { children[*rows[i].LayerResourceParentID] = rows[i] @@ -417,6 +428,10 @@ func (c *converter) indexDiskStacks() { continue } + if diskless[replicaKey{node: row.NodeName, rd: row.ResourceName}] { + continue + } + var stack []string // Bounded by the row count, so a corrupt chain that loops ends. diff --git a/pkg/linstormigrate/layer_stack_test.go b/pkg/linstormigrate/layer_stack_test.go index c4cc5dba..908005db 100644 --- a/pkg/linstormigrate/layer_stack_test.go +++ b/pkg/linstormigrate/layer_stack_test.go @@ -124,6 +124,35 @@ func TestSnapshotAndSuffixedLayersDoNotShapeTheStack(t *testing.T) { } } +// A diskless replica (a client or a tiebreaker) is built as DRBD over +// STORAGE whatever its definition's stack, so it neither shapes the stack +// nor makes the diskful replicas look divergent. +func TestADisklessReplicaDoesNotShapeTheStack(t *testing.T) { + t.Parallel() + + for name, rdStack := range map[string]string{"empty": "[]", "named": `["DRBD","LUKS","STORAGE"]`} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dump := stackDump(rdStack, + layerChain(1, "NODE-A", "PVC-S", "DRBD", "LUKS", "STORAGE"), layerChain(10, "NODE-B", "PVC-S", "DRBD", "STORAGE")) + dump.Resources = []ResourceRow{ + {NodeName: "NODE-A", ResourceName: "PVC-S", UUID: "r-a"}, + {NodeName: "NODE-B", ResourceName: "PVC-S", ResourceFlags: resourceFlagDiskless, UUID: "r-b"}, + } + + got, res := convertedStack(t, dump) + if !slices.Equal(got, []string{"DRBD", "LUKS", "STORAGE"}) { + t.Fatalf("stack %v; warnings %v", got, res.Warnings) + } + + if hasWarning(res, "built as") { + t.Errorf("a diskless replica was reported as divergent; warnings %v", res.Warnings) + } + }) + } +} + // Replicas built two ways leave nothing to read: the stack stays unset // and the operator is told which ways. func TestReplicasThatDisagreeLeaveTheStackUnset(t *testing.T) { From 52545a6d7dc7896a45106e0c18a92e94aa2bf089 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Wed, 14 Oct 2026 03:57:20 +0500 Subject: [PATCH 13/22] docs(layer-stack): say a pin is flagged only when it is short of the group Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- internal/controller/resourcedefinition_layer_stack.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index eab7ddfc..c1ef5ebc 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -32,7 +32,7 @@ The default is `["DRBD","STORAGE"]`, used when neither the RD nor its resource g An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The REST create and `rg spawn` paths copy the group's stack onto the RD when it is created. An RD that ends up with an empty stack in a group that sets one (created with `kubectl apply` or GitOps, or moved by `rd modify --resource-group`) is pinned by the controller, and the satellite builds nothing for it until then: - An RD that holds no data yet gets the group's stack. -- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. The group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. +- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack only when the source set one by choice, and otherwise is left for the controller to pin from its own group. diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index bcdb233d..325e0359 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -105,7 +105,7 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl } logf.FromContext(ctx).Info("pinned layer stack", "rd", rd.Name, "stack", stack, "group", rg.Name, - "groupStack", groupStack, "groupStackApplied", !unapplied) + "groupStack", groupStack, "shortOfGroup", unapplied) if unapplied && r.Recorder != nil { r.Recorder.Eventf(rd, nil, corev1.EventTypeWarning, layerStackNotInheritedReason, layerStackPinAction, From c717fdb865cc109519e6b29c037d122f9860f7c3 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 23 Oct 2026 21:06:21 +0200 Subject: [PATCH 14/22] fix(lvm): wipe the signatures a thick LV's recycled extents held A thick LV is allocated with -Wn -Zn, without zeroing, and nothing wiped a volume's signatures when it was deleted, so a new LV starts with whatever its extents held. When that was a deleted LUKS volume's header, the LUKS layer found a header, skipped the format and opened it with the cluster passphrase every volume shares: the new volume showed the old one's contents. Any other leftover, a filesystem or DRBD metadata, made the LUKS layer refuse to format a volume that holds nothing. CreateVolume now creates the LV under a temporary name, the volume's name with a .bsnew suffix, runs wipefs over it and only then renames it to the volume's name, so nothing under a volume's name carries a signature another volume left. wipefs erases the signatures libblkid recognises, the LUKS header magic among them, and nothing else: the rest of the old volume's blocks stays on the extents, and a plaintext volume can read it back from the raw device. Zeroing the LV would close that, at the cost of writing a large thick LV in full before it can be used. An LV already under the volume's name is the volume and is never wiped. A temporary LV left by a run that stopped before the rename is wiped and renamed rather than created again. A failed listing, wipe or rename fails CreateVolume, and a failed wipe or rename leaves nothing under the volume's name. The orphan sweeper only knows LVs named like a volume, so it never sees a temporary one; DeleteVolume removes a leftover one along with the volume. DeleteVolume wipes nothing: lvremove refuses an LV that is still open, and that LV has to keep its data. The next CreateVolume over the freed extents wipes their signatures anyway. Thin LVM, ZFS and the file provider do not need this: a fresh thin LV, zvol or backing file reads zeroes where nothing has been written. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- pkg/storage/lvm/lvm_common.go | 10 +- pkg/storage/lvm/lvm_thick.go | 125 ++++++++++++++++---- pkg/storage/lvm/lvm_thick_test.go | 6 +- pkg/storage/lvm/thick_wipe_test.go | 180 +++++++++++++++++++++++++++++ 5 files changed, 292 insertions(+), 31 deletions(-) create mode 100644 pkg/storage/lvm/thick_wipe_test.go diff --git a/docs/layer-stack.md b/docs/layer-stack.md index c1ef5ebc..405a3c97 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -38,7 +38,7 @@ Clones and restores copy bytes, so the RD they create takes the stack those byte Changing a group's stack later never reshapes RDs that already carry one, as in upstream LINSTOR, where the stack is decided when a resource is created. -As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. On a fresh thick LV that reuses extents of an earlier volume the signature can be stale; clear it with `wipefs -a ` once you have checked it holds nothing you need. +As a last guard, the LUKS layer refuses to `luksFormat` a device that already carries a signature `blkid -p` recognises (DRBD metadata, a filesystem, a partition table). The replica's apply fails with `luks: device already holds data` and names what it found. The satellite wipes a thick LV it creates before the volume takes its name, so a fresh volume never inherits a signature from extents an earlier volume used; one created outside blockstor may, so clear it with `wipefs -a ` once you have checked it holds nothing you need. Find the RDs short of a layer their group asks for: diff --git a/pkg/storage/lvm/lvm_common.go b/pkg/storage/lvm/lvm_common.go index c1357771..187a47d2 100644 --- a/pkg/storage/lvm/lvm_common.go +++ b/pkg/storage/lvm/lvm_common.go @@ -165,6 +165,12 @@ const RestoreIncompleteTag = "blockstor-restore-incomplete" // `blockstor-restore-incomplete-v2` would have matched a bare // `strings.Contains` check). func lvHasRestoreIncompleteTag(ctx context.Context, ex storage.Exec, vg, lvName string) bool { + return lvHasTag(ctx, ex, vg, lvName, RestoreIncompleteTag) +} + +// lvHasTag reports whether the named LV carries tag, matching whole +// tokens of the comma-separated `lv_tags` list. Errors fold into false. +func lvHasTag(ctx context.Context, ex storage.Exec, vg, lvName, tag string) bool { ctx, cancel := withProbeTimeout(ctx) defer cancel() @@ -176,8 +182,8 @@ func lvHasRestoreIncompleteTag(ctx context.Context, ex storage.Exec, vg, lvName return false } - for tag := range strings.SplitSeq(strings.TrimSpace(string(out)), ",") { - if strings.TrimSpace(tag) == RestoreIncompleteTag { + for got := range strings.SplitSeq(strings.TrimSpace(string(out)), ",") { + if strings.TrimSpace(got) == tag { return true } } diff --git a/pkg/storage/lvm/lvm_thick.go b/pkg/storage/lvm/lvm_thick.go index a1b62bbc..773aa2b5 100644 --- a/pkg/storage/lvm/lvm_thick.go +++ b/pkg/storage/lvm/lvm_thick.go @@ -61,33 +61,77 @@ func (*Thick) Kind() string { return "LVM" } // integrity). See storage.ResizeZeroFiller. func (*Thick) ResizeZeroFills() bool { return false } -// CreateVolume allocates a thick LV. Idempotent: existing LV is a no-op. +// CreateVolume allocates a thick LV with nothing on it. Idempotent: an LV +// already under the volume's name is the volume and is left alone. +// +// A thick LV is allocated without zeroing (-Wn -Zn, below), so it starts +// with whatever the extents held: the filesystem, DRBD metadata or LUKS +// header of a volume deleted earlier. Left there, a LUKS header is opened +// with the cluster passphrase every volume shares and hands the new +// volume the old one's contents, and any other signature makes the LUKS +// layer refuse to format a volume that holds nothing. So the LV is +// created under a temporary name (freshLVName), wiped there, and only +// then renamed to the volume's name: nothing ever finds an unwiped LV +// under a name it treats as a volume, this satellite after a crash or an +// older one after a rollback. A temporary LV left by a crash is wiped +// and renamed by the next call; it was never exposed. func (t *Thick) CreateVolume(ctx context.Context, vol storage.Volume) error { - if t.lvExists(ctx, volumeLVName(vol)) { + final := volumeLVName(vol) + fresh := freshLVName(final) + + names, err := t.lvNames(ctx) + if err != nil { + return err + } + + if names[final] { return nil } - sizeMiB := max(vol.SizeKib/mibPerKib, 1) + if !names[fresh] { + sizeMiB := max(vol.SizeKib/mibPerKib, 1) + + // Skip the optional zero-on-create step (-Wn -Zn) — the satellite + // container has no udev daemon, and the wipe trips on the missing + // /dev// symlink. Same trick as install-pools.sh. Bug 305: + // the udev-less activation section rides in the SAME single + // `--config` as the device filter (ArgsUdevless) — a separate + // `--config activation{…}` would be a repeated `--config`, which + // the LVM CLI rejects. + _, err = t.exec.Run(ctx, "lvcreate", + ArgsUdevless("--size", strconv.FormatInt(sizeMiB, 10)+"MiB", + "--name", fresh, + "-Wn", "-Zn", + t.cfg.VolumeGroup)...) + if err != nil { + return errors.Wrapf(err, "lvcreate %s", fresh) + } + } - // Skip the optional zero-on-create step (-Wn -Zn) — the satellite - // container has no udev daemon, and the wipe trips on the missing - // /dev// symlink. Same trick as install-pools.sh. Bug 305: - // the udev-less activation section rides in the SAME single - // `--config` as the device filter (ArgsUdevless) — a separate - // `--config activation{…}` would be a repeated `--config`, which - // the LVM CLI rejects. - _, err := t.exec.Run(ctx, "lvcreate", - ArgsUdevless("--size", strconv.FormatInt(sizeMiB, 10)+"MiB", - "--name", volumeLVName(vol), - "-Wn", "-Zn", - t.cfg.VolumeGroup)...) + device := "/dev/" + t.cfg.VolumeGroup + "/" + fresh + + _, err = t.exec.Run(ctx, "wipefs", "--all", "--force", device) + if err != nil { + return errors.Wrapf(err, "wipefs %s", device) + } + + _, err = t.exec.Run(ctx, "lvrename", + ArgsUdevless(t.cfg.VolumeGroup, fresh, final)...) if err != nil { - return errors.Wrapf(err, "lvcreate %s", volumeLVName(vol)) + return errors.Wrapf(err, "lvrename %s to %s", fresh, final) } return nil } +// freshLVName is where CreateVolume allocates and wipes an LV before it +// takes the volume's name. The suffix after the last underscore is not +// the five-digit volume number, so neither the orphan sweeper's +// listing nor a lookup of the volume's own name ever matches it. +func freshLVName(final string) string { + return final + ".bsnew" +} + // ResizeVolume grows the LV to vol.SizeKib. Bug 269 (P1): inherits // the udev-less workaround CreateVolume uses; satellite container // has no udev daemon so `activation{udev_sync=0 udev_rules=0}` keeps @@ -112,17 +156,22 @@ func (t *Thick) ResizeVolume(ctx context.Context, vol storage.Volume) error { return nil } -// DeleteVolume idempotently removes the LV. +// DeleteVolume idempotently removes the LV, and a temporary LV a crashed +// CreateVolume left for it. Nothing is wiped here: an LV that lvremove +// refuses, because it is still open, keeps its data, and the next +// CreateVolume over these extents wipes them anyway. func (t *Thick) DeleteVolume(ctx context.Context, vol storage.Volume) error { - if !t.lvExists(ctx, volumeLVName(vol)) { - return nil - } + for _, name := range []string{volumeLVName(vol), freshLVName(volumeLVName(vol))} { + if !t.lvExists(ctx, name) { + continue + } - _, err := t.exec.Run(ctx, "lvremove", - Args("--force", - t.cfg.VolumeGroup+"/"+volumeLVName(vol))...) - if err != nil { - return errors.Wrapf(err, "lvremove %s", volumeLVName(vol)) + _, err := t.exec.Run(ctx, "lvremove", + Args("--force", + t.cfg.VolumeGroup+"/"+name)...) + if err != nil { + return errors.Wrapf(err, "lvremove %s", name) + } } return nil @@ -349,6 +398,32 @@ func (t *Thick) copyAndClearSentinel(ctx context.Context, srcName, tgtName strin return nil } +// lvNames lists every LV in the VG. Unlike lvExists it does not fold an +// error into "absent": CreateVolume decides from it whether an LV is a +// volume it must leave alone. +func (t *Thick) lvNames(ctx context.Context) (map[string]bool, error) { + ctx, cancel := withProbeTimeout(ctx) + defer cancel() + + out, err := t.exec.Run(ctx, "lvs", + Args("--noheadings", + "-o", "lv_name", + t.cfg.VolumeGroup)...) + if err != nil { + return nil, errors.Wrapf(err, "lvs %s", t.cfg.VolumeGroup) + } + + names := make(map[string]bool) + + for line := range strings.SplitSeq(string(out), "\n") { + if name := strings.TrimSpace(line); name != "" { + names[name] = true + } + } + + return names, nil +} + // lvExists is the same idempotency primitive Thin uses. Bug 270: // bounded ctx so a stuck `lvs` cannot block the create/delete path. func (t *Thick) lvExists(ctx context.Context, lvName string) bool { diff --git a/pkg/storage/lvm/lvm_thick_test.go b/pkg/storage/lvm/lvm_thick_test.go index f8f09a7a..0a2d7ea8 100644 --- a/pkg/storage/lvm/lvm_thick_test.go +++ b/pkg/storage/lvm/lvm_thick_test.go @@ -43,7 +43,7 @@ func TestThickKind(t *testing.T) { // /dev symlinks. func TestThickCreateVolumeIssuesLvcreate(t *testing.T) { fx := storage.NewFakeExec() - fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg/pvc-1_00000", + fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg", storage.FakeResponse{Stdout: []byte("")}) p := lvm.NewThick(lvm.ThickConfig{VolumeGroup: "vg"}, fx) @@ -59,7 +59,7 @@ func TestThickCreateVolumeIssuesLvcreate(t *testing.T) { // Bug 305: device filter + udev-less activation ride in ONE // `--config` string (LVM rejects a repeated `--config`). - want := "lvcreate --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } activation { udev_sync=0 udev_rules=0 } --size 1024MiB --name pvc-1_00000 -Wn -Zn vg" + want := "lvcreate --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } activation { udev_sync=0 udev_rules=0 } --size 1024MiB --name pvc-1_00000.bsnew -Wn -Zn vg" if !slices.Contains(fx.CommandLines(), want) { t.Errorf("expected %q in calls; got %v", want, fx.CommandLines()) } @@ -68,7 +68,7 @@ func TestThickCreateVolumeIssuesLvcreate(t *testing.T) { // TestThickCreateVolumeIdempotent: existing LV → no lvcreate. func TestThickCreateVolumeIdempotent(t *testing.T) { fx := storage.NewFakeExec() - fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg/pvc-1_00000", + fx.Expect("lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg", storage.FakeResponse{Stdout: []byte("pvc-1_00000\n")}) p := lvm.NewThick(lvm.ThickConfig{VolumeGroup: "vg"}, fx) diff --git a/pkg/storage/lvm/thick_wipe_test.go b/pkg/storage/lvm/thick_wipe_test.go new file mode 100644 index 00000000..1374cacd --- /dev/null +++ b/pkg/storage/lvm/thick_wipe_test.go @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 + +package lvm_test + +import ( + "errors" + "slices" + "testing" + + "github.com/cozystack/blockstor/pkg/storage" + "github.com/cozystack/blockstor/pkg/storage/lvm" +) + +const ( + lvsVG = "lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg" + lvsFinal = "lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg/pvc-1_00000" + lvsFresh = "lvs --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --noheadings -o lv_name vg/pvc-1_00000.bsnew" + wipeFresh = "wipefs --all --force /dev/vg/pvc-1_00000.bsnew" + wipeFinal = "wipefs --all --force /dev/vg/pvc-1_00000" + renameLV = "lvrename --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } activation { udev_sync=0 udev_rules=0 } vg pvc-1_00000.bsnew pvc-1_00000" + lvremove = "lvremove --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --force vg/pvc-1_00000" + lvcreateLV = "lvcreate --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } activation { udev_sync=0 udev_rules=0 } --size 1024MiB --name pvc-1_00000.bsnew -Wn -Zn vg" +) + +var ( + errWipeBusy = errors.New("wipefs: device busy") + errLvsLock = errors.New("lvs: lock timeout") + errRename = errors.New("lvrename: busy") + errInUse = errors.New("logical volume vg/pvc-1_00000 in use") +) + +func thickVol() storage.Volume { + return storage.Volume{ResourceName: "pvc-1", VolumeNumber: 0, SizeKib: 1024 * 1024} +} + +func create(t *testing.T, fx *storage.FakeExec) error { + t.Helper() + + return lvm.NewThick(lvm.ThickConfig{VolumeGroup: "vg"}, fx).CreateVolume(t.Context(), thickVol()) //nolint:wrapcheck // test helper +} + +// A thick LV is allocated without zeroing, so it starts with what its +// extents held. It is created under a temporary name, wiped there, and only +// then renamed to the volume's, so no stale LUKS header or filesystem is +// ever found under a volume's name. +func TestThickCreateVolumeWipesTheFreshLVBeforeItTakesTheVolumesName(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte(" other_00000\n")}) + + if err := create(t, fx); err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + lines := fx.CommandLines() + c, w, r := slices.Index(lines, lvcreateLV), slices.Index(lines, wipeFresh), slices.Index(lines, renameLV) + + if c < 0 || w < 0 || r < 0 || c >= w || w >= r { + t.Errorf("want lvcreate of the temporary LV, then wipefs, then lvrename; got %v", lines) + } +} + +// An LV under the volume's name is the volume. It is never wiped, whatever +// it holds. +func TestThickCreateVolumeLeavesAnLVUnderTheVolumesNameAlone(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte(" pvc-1_00000\n")}) + + if err := create(t, fx); err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + for _, l := range fx.CommandLines() { + if l == wipeFinal || l == wipeFresh || l == lvcreateLV || l == renameLV { + t.Errorf("touched an existing volume: %v", fx.CommandLines()) + } + } +} + +// A crash between the create and the rename leaves the temporary LV. It was +// never exposed under the volume's name, so the next call wipes it and +// renames it rather than creating another. +func TestThickCreateVolumeFinishesATemporaryLVACrashLeft(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte(" pvc-1_00000.bsnew\n")}) + + if err := create(t, fx); err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + lines := fx.CommandLines() + if slices.Contains(lines, lvcreateLV) { + t.Errorf("created a second LV over the leftover: %v", lines) + } + + w, r := slices.Index(lines, wipeFresh), slices.Index(lines, renameLV) + if w < 0 || r < 0 || w >= r { + t.Errorf("want the leftover wiped, then renamed; got %v", lines) + } +} + +// A listing that fails says nothing about whether the volume exists, so the +// create stops rather than assume either way. +func TestThickCreateVolumeStopsWhenTheListingFails(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Err: errLvsLock}) + + if err := create(t, fx); err == nil { + t.Fatal("CreateVolume succeeded over a failed listing") + } + + if len(fx.CommandLines()) != 1 { + t.Errorf("ran more than the listing: %v", fx.CommandLines()) + } +} + +// A wipe that fails leaves the LV under its temporary name, so it is not +// handed on, and the next attempt wipes again. +func TestThickCreateVolumeDoesNotRenameOverAFailedWipe(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte("")}) + fx.Expect(wipeFresh, storage.FakeResponse{Err: errWipeBusy}) + + if err := create(t, fx); err == nil { + t.Fatal("CreateVolume succeeded over a failed wipe") + } + + if slices.Contains(fx.CommandLines(), renameLV) { + t.Errorf("renamed an LV whose wipe failed: %v", fx.CommandLines()) + } +} + +// A rename that fails is an error: the volume does not exist under its name. +func TestThickCreateVolumeFailsOnAFailedRename(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte("")}) + fx.Expect(renameLV, storage.FakeResponse{Err: errRename}) + + if err := create(t, fx); err == nil { + t.Fatal("CreateVolume succeeded over a failed rename") + } +} + +// Delete wipes nothing. lvremove refuses an LV that is still open, and that +// LV has to keep its data; the next CreateVolume over these extents wipes +// them anyway. +func TestThickDeleteVolumeWipesNothingEvenWhenTheLVIsInUse(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsFinal, storage.FakeResponse{Stdout: []byte("pvc-1_00000\n")}) + fx.Expect(lvremove, storage.FakeResponse{Err: errInUse}) + + err := lvm.NewThick(lvm.ThickConfig{VolumeGroup: "vg"}, fx).DeleteVolume(t.Context(), thickVol()) + if err == nil { + t.Fatal("DeleteVolume succeeded over a refused lvremove") + } + + for _, l := range fx.CommandLines() { + if len(l) >= 6 && l[:6] == "wipefs" { + t.Errorf("wiped an LV on delete: %v", fx.CommandLines()) + } + } +} + +// Delete also removes a temporary LV a crashed create left for this volume, +// so its extents go back to the VG. The positive control for the test above: +// a removable LV is removed. +func TestThickDeleteVolumeRemovesALeftoverTemporaryLV(t *testing.T) { + fx := storage.NewFakeExec() + fx.Expect(lvsFinal, storage.FakeResponse{Stdout: []byte("pvc-1_00000\n")}) + fx.Expect(lvsFresh, storage.FakeResponse{Stdout: []byte("pvc-1_00000.bsnew\n")}) + + if err := lvm.NewThick(lvm.ThickConfig{VolumeGroup: "vg"}, fx).DeleteVolume(t.Context(), thickVol()); err != nil { + t.Fatalf("DeleteVolume: %v", err) + } + + lines := fx.CommandLines() + if !slices.Contains(lines, lvremove) || + !slices.Contains(lines, "lvremove --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --force vg/pvc-1_00000.bsnew") { + t.Errorf("want both LVs removed; got %v", lines) + } +} From 9c3b2f60849f32c85b1972482f43b10b8c332de5 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 23 Oct 2026 21:15:50 +0200 Subject: [PATCH 15/22] fix(controller,satellite): find a definition's group under its folded name A definition keeps its group's name as the operator spelled it (`--resource-group DfltRscGrp`), while the group's CR is stored under the lowercased name. The witness gate and the effective-props resolver (which the satellite, the dispatcher and the controller render DRBD options with) looked the group up by the spelled name, missed it, and went on as if there were no group: a STORAGE-only group got a DRBD witness, and the group's DRBD options were left out of the rendered config. Both now look the group up under the folded name, as the store does. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- .../resourcedefinition_controller.go | 3 +- .../tiebreaker_no_drbd_bug_334_test.go | 16 ++++++++- pkg/effectiveprops/effective.go | 5 ++- pkg/effectiveprops/effective_test.go | 33 +++++++++++++++++++ 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/internal/controller/resourcedefinition_controller.go b/internal/controller/resourcedefinition_controller.go index a0fc56da..a6a82f29 100644 --- a/internal/controller/resourcedefinition_controller.go +++ b/internal/controller/resourcedefinition_controller.go @@ -43,6 +43,7 @@ import ( blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" apiv1 "github.com/cozystack/blockstor/pkg/api/v1" "github.com/cozystack/blockstor/pkg/store" + "github.com/cozystack/blockstor/pkg/store/k8s" ) // Bug148ResizePendingAnnotationPrefix is the per-volume annotation @@ -991,7 +992,7 @@ func (r *ResourceDefinitionReconciler) resolveRDLayerStack(ctx context.Context, reader := r.directOrCached() var rg blockstoriov1alpha1.ResourceGroup - if err := reader.Get(ctx, client.ObjectKey{Name: rd.Spec.ResourceGroupName}, &rg); err != nil { + if err := reader.Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rg); err != nil { return apiv1.DefaultLayerStack() } diff --git a/internal/controller/tiebreaker_no_drbd_bug_334_test.go b/internal/controller/tiebreaker_no_drbd_bug_334_test.go index a9587b5d..bdb1f93d 100644 --- a/internal/controller/tiebreaker_no_drbd_bug_334_test.go +++ b/internal/controller/tiebreaker_no_drbd_bug_334_test.go @@ -194,9 +194,23 @@ func TestTieBreakerStillSpawnedForDRBDStackRD(t *testing.T) { // production code path where the RD itself carries no LayerStack and // the effective stack is inherited from a parent RG. When the parent // RG specifies a STORAGE-only stack the witness must still be skipped. +// +// The RD may name its group in another case than the CR's folded name, as +// LINSTOR names are case-insensitive; the group is found either way. func TestNoTieBreakerSpawnedForEmptyLayerStackInheritsRG(t *testing.T) { t.Parallel() + for _, group := range []string{"rg-storage", "RG-Storage"} { + t.Run(group, func(t *testing.T) { + t.Parallel() + noTieBreakerUnderAStorageOnlyGroup(t, group) + }) + } +} + +func noTieBreakerUnderAStorageOnlyGroup(t *testing.T, group string) { + t.Helper() + scheme := newScheme(t) st := store.NewInMemory() ctx := context.Background() @@ -229,7 +243,7 @@ func TestNoTieBreakerSpawnedForEmptyLayerStackInheritsRG(t *testing.T) { rd := &blockstoriov1alpha1.ResourceDefinition{ ObjectMeta: metav1.ObjectMeta{Name: "rd-inherit"}, Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ - ResourceGroupName: "rg-storage", + ResourceGroupName: group, }, } diff --git a/pkg/effectiveprops/effective.go b/pkg/effectiveprops/effective.go index 5e7ed113..6cc0952c 100644 --- a/pkg/effectiveprops/effective.go +++ b/pkg/effectiveprops/effective.go @@ -43,6 +43,7 @@ import ( blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" "github.com/cozystack/blockstor/pkg/drbd" + "github.com/cozystack/blockstor/pkg/store/k8s" ) // Resolve walks the four scopes and returns the merged Props @@ -214,7 +215,9 @@ func scopeInputs(ctx context.Context, c client.Reader, rd *blockstoriov1alpha1.R var rgObj blockstoriov1alpha1.ResourceGroup - getErr := c.Get(ctx, client.ObjectKey{Name: rd.Spec.ResourceGroupName}, &rgObj) + // The RD keeps the group's name as the operator spelled it; the CR is + // stored under the folded one. + getErr := c.Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rgObj) switch { case getErr == nil: rgInfo = scopeInfo{ diff --git a/pkg/effectiveprops/effective_test.go b/pkg/effectiveprops/effective_test.go index ae2608b1..093246af 100644 --- a/pkg/effectiveprops/effective_test.go +++ b/pkg/effectiveprops/effective_test.go @@ -205,6 +205,39 @@ func TestResolve_RGOverridesController_RDOverridesRG(t *testing.T) { } } +// TestResolve_RGFoundUnderAnotherCase: LINSTOR names are case-insensitive, +// and an RD keeps its group's name as the operator spelled it while the +// group's CR is stored under the folded one. The group's options still +// apply. +func TestResolve_RGFoundUnderAnotherCase(t *testing.T) { + rg := &blockstoriov1alpha1.ResourceGroup{ + ObjectMeta: v1.ObjectMeta{Name: "rg-mixed"}, + Spec: blockstoriov1alpha1.ResourceGroupSpec{ + DRBDOptions: &blockstoriov1alpha1.DRBDOptions{ + Net: &blockstoriov1alpha1.DRBDNetOptions{MaxBuffers: int32Ptr(2000)}, + }, + }, + } + + rd := &blockstoriov1alpha1.ResourceDefinition{ + ObjectMeta: v1.ObjectMeta{Name: "rd1"}, + Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ResourceGroupName: "RG-Mixed"}, + } + + res := &blockstoriov1alpha1.Resource{ObjectMeta: v1.ObjectMeta{Name: "rd1-node1"}} + cli := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(rg).Build() + + got, err := effectiveprops.Resolve(context.Background(), client.Reader(cli), res, rd) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + + if got["DrbdOptions/Net/max-buffers"] != "2000" { + t.Fatalf("the group's option under a group named in another case: got max-buffers=%q, want 2000", + got["DrbdOptions/Net/max-buffers"]) + } +} + // TestResolve_ControllerNonDRBDPropDropped pins that a controller-level // non-DRBD ExtraProp (something outside the DrbdOptions/ namespace) does // NOT leak onto the resource's effective props — only the resource's own From d889dafa84ef4dd8f8c06943a35de1ab2da2d426 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Fri, 23 Oct 2026 21:25:00 +0200 Subject: [PATCH 16/22] test(layer-stack): pin a group's stack at operator-CLI level An L6 cell and an L7 replay drive the layer-stack pin through the linstor CLI: a definition with nothing on disk, created in a group without a stack, has none, and takes the LUKS group's stack once rd modify moves it there; the REST create copies a group's stack itself, so only the move shows the controller's pin. One with data keeps DRBD,STORAGE when its group gains LUKS, is annotated with the group's stack, and gets the LayerStackNotInherited Event; a clone taken straight after the group change is refused as undecided until the pin and then succeeds with the source's stack. The cell also checks the data reads back unchanged. The replay runner gains two await kinds to read what the CLI does not show: rd_layer_stack (spec.layerStack) and rd_annotation. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- CLAUDE.md | 2 +- tests/e2e/cli-matrix/rg-layer-stack-pin.sh | 195 ++++++++++++++++++ tests/operator-harness/lib.sh | 32 +++ tests/operator-harness/replay-runner.sh | 5 + .../replay/rg-layer-stack-pin.yaml | 139 +++++++++++++ 5 files changed, 372 insertions(+), 1 deletion(-) create mode 100755 tests/e2e/cli-matrix/rg-layer-stack-pin.sh create mode 100644 tests/operator-harness/replay/rg-layer-stack-pin.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 329841ad..815803ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ Copy the closest existing YAML and fill: - `teardown[]` — cleanup CLI invocations. - `invariants[]` — currently only `no_orphans` is implemented. -Available `await.kind` values: `replica_count`, `disk_state`, `all_uptodate`, `replica_diskless`, `no_tiebreaker`, `sync_clean`, `resource_absent`, `rd_absent`, `vd_size_kib`, `vd_count`, `pvc_capacity`, `pod_md5_invariant`. See the header comment of `tests/operator-harness/replay-runner.sh` for the contract. +Available `await.kind` values: `replica_count`, `disk_state`, `all_uptodate`, `replica_diskless`, `no_tiebreaker`, `sync_clean`, `resource_absent`, `rd_absent`, `vd_size_kib`, `vd_count`, `rd_layer_stack`, `rd_annotation`, `pvc_capacity`, `pod_md5_invariant`. See the header comment of `tests/operator-harness/replay-runner.sh` for the contract. ## Running the harness diff --git a/tests/e2e/cli-matrix/rg-layer-stack-pin.sh b/tests/e2e/cli-matrix/rg-layer-stack-pin.sh new file mode 100755 index 00000000..53aed290 --- /dev/null +++ b/tests/e2e/cli-matrix/rg-layer-stack-pin.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# +# usage: rg-layer-stack-pin.sh WORK_DIR +# +# L6 cli-matrix cell — a definition's layer stack is decided once, from +# its resource group, and never reshapes a volume that already exists. +# +# The controller writes the group's stack onto a definition that leaves +# it empty (internal/controller/resourcedefinition_layer_stack.go). A +# definition that holds no data takes the group's stack; one that already +# went to disk under the default stack keeps it, carries the +# blockstor.io/unapplied-group-layer-stack annotation and gets a +# LayerStackNotInherited warning Event, so a LUKS group added over a +# plaintext volume never has the satellite luksFormat that volume. +# +# Steps: +# a) rg c (no stack) and rg c --layer-list +# drbd,luks,storage; rd c in stays without a stack; rd +# modify --resource-group → spec.layerStack becomes +# DRBD,LUKS,STORAGE. The REST create copies a group's stack itself, +# so only a move into the group shows the controller's pin. +# b) rd c + vd c + r c --auto-place=2 in ; write data. +# c) rg modify --layer-list drbd,luks,storage. +# d) rd clone of the data RD straight after (c). Inside the window before +# the controller pins it the clone is refused, saying the stack is not +# pinned yet; it then succeeds. A stand fast enough to pin before the +# clone lands sees the success directly. +# e) the data RD's spec.layerStack stays DRBD,STORAGE, the annotation +# names the group's stack, the Event is there, the data reads back +# unchanged; the clone has DRBD,STORAGE too. + +set -euo pipefail + +WORK_DIR=${1:?work_dir required} +export KUBECONFIG="$WORK_DIR/kubeconfig" + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=lib.sh +source "$SCRIPT_DIR/lib.sh" + +require_workers 2 + +linstor_cli_setup + +LUKS_RG=cli-matrix-ls-luks +PLAIN_RG=cli-matrix-ls-plain +RD_FRESH=cli-matrix-ls-fresh +RD_DATA=cli-matrix-ls-data +RD_CLONE=cli-matrix-ls-clone +POOL=${POOL:-lvm-thin} +PASSPHRASE='cli-matrix-ls-pp!' +ANNOTATION=blockstor.io/unapplied-group-layer-stack + +cleanup() { + delete_rd "$RD_CLONE" + delete_rd "$RD_DATA" + delete_rd "$RD_FRESH" + "${LCTL[@]}" resource-group delete "$PLAIN_RG" >/dev/null 2>&1 || true + "${LCTL[@]}" resource-group delete "$LUKS_RG" >/dev/null 2>&1 || true + assert_no_orphans "$RD_DATA" + assert_no_orphans "$RD_CLONE" + cleanup_encryption_state + linstor_cli_teardown +} +trap cleanup EXIT + +cleanup_encryption_state + +# rd_layer_stack RD — the stack the CRD carries, comma-joined. +rd_layer_stack() { + kubectl get "resourcedefinitions.blockstor.cozystack.io/$1" \ + -o jsonpath='{.spec.layerStack}' 2>/dev/null \ + | jq -r 'join(",") | ascii_upcase' 2>/dev/null || echo "" +} + +# wait_layer_stack RD EXPECTED [TIMEOUT=60] +wait_layer_stack() { + local rd=$1 expected=$2 timeout=${3:-60} got="" + local deadline=$(( $(date +%s) + timeout )) + while (( $(date +%s) < deadline )); do + got=$(rd_layer_stack "$rd") + [[ "$got" == "$expected" ]] && return 0 + sleep 2 + done + echo "FAIL: $rd layer stack '$got', want '$expected' within ${timeout}s" >&2 + return 1 +} + +echo ">> pre-flight: 2 healthy $POOL SPs" +sp_json=$("${LCTL[@]}" --machine-readable storage-pool list --storage-pools "$POOL" 2>/dev/null || echo "[]") +ok_nodes=$(jq -r '[.[]? | .[]? | select(.provider_kind != null) | .node_name] | unique | length' <<<"$sp_json" 2>/dev/null || echo 0) +if (( ok_nodes < 2 )); then + echo "SKIP: $POOL SP not on >=2 nodes (got $ok_nodes)" + exit 0 +fi + +"${LCTL[@]}" encryption create-passphrase --passphrase "$PASSPHRASE" >/dev/null 2>&1 || true +"${LCTL[@]}" controller set-property DrbdOptions/EncryptPassphrase "$PASSPHRASE" >/dev/null 2>&1 || true + +echo ">> a) an RD with nothing on disk moved into a LUKS group takes its stack" +"${LCTL[@]}" resource-group create "$PLAIN_RG" --storage-pool "$POOL" >/dev/null +"${LCTL[@]}" resource-group create "$LUKS_RG" --layer-list drbd,luks,storage --storage-pool "$POOL" >/dev/null +"${LCTL[@]}" resource-definition create --resource-group "$PLAIN_RG" "$RD_FRESH" >/dev/null +sleep 10 +fresh_stack=$(rd_layer_stack "$RD_FRESH") +if [[ -n "$fresh_stack" ]]; then + echo "FAIL: $RD_FRESH in a group without a stack has '$fresh_stack', want none" >&2 + exit 1 +fi +# Nothing on the REST side copies the stack on a modify: the controller's +# pin does. +"${LCTL[@]}" resource-definition modify --resource-group "$LUKS_RG" "$RD_FRESH" >/dev/null +wait_layer_stack "$RD_FRESH" "DRBD,LUKS,STORAGE" + +echo ">> b) an RD with data under a group that sets no stack" +"${LCTL[@]}" resource-definition create --resource-group "$PLAIN_RG" "$RD_DATA" >/dev/null +"${LCTL[@]}" volume-definition create "$RD_DATA" 64M >/dev/null +"${LCTL[@]}" resource create --auto-place=2 --storage-pool="$POOL" "$RD_DATA" >/dev/null + +deadline=$(( $(date +%s) + 60 )) +placed=() +while (( $(date +%s) < deadline )); do + mapfile -t placed < <(linstor_diskful_nodes "$RD_DATA") + if (( ${#placed[@]} == 2 )); then break; fi + sleep 2 +done +if (( ${#placed[@]} != 2 )); then + echo "FAIL: $RD_DATA did not autoplace 2 replicas" >&2 + exit 1 +fi +N1="${placed[0]}" +N2="${placed[1]}" + +wait_uptodate "$RD_DATA" "$N1" "$N2" +DEV=$(device_for_rd "$RD_DATA" "$N1") +RD=$RD_DATA +md5_before=$(write_random "$N1" "$DEV" 262144) + +echo ">> c) rg modify --layer-list drbd,luks,storage over the group holding data" +"${LCTL[@]}" resource-group modify "$PLAIN_RG" --layer-list drbd,luks,storage >/dev/null + +echo ">> d) clone the data RD while its stack may still be undecided" +refused=0 +deadline=$(( $(date +%s) + 60 )) +err_file=$(mktemp) +until "${LCTL[@]}" resource-definition clone "$RD_DATA" "$RD_CLONE" >/dev/null 2>"$err_file"; do + if ! grep -q "not pinned yet" "$err_file"; then + echo "FAIL: clone refused for another reason than the undecided stack:" >&2 + cat "$err_file" >&2 + rm -f "$err_file" + exit 1 + fi + refused=1 + if (( $(date +%s) >= deadline )); then + echo "FAIL: clone still refused as undecided after 60s" >&2 + rm -f "$err_file" + exit 1 + fi + sleep 2 +done +rm -f "$err_file" +echo " clone refused as undecided before the pin: $refused" + +echo ">> e) the data RD keeps the stack its data has" +wait_layer_stack "$RD_DATA" "DRBD,STORAGE" + +annotation=$(kubectl get "resourcedefinitions.blockstor.cozystack.io/$RD_DATA" -o json 2>/dev/null \ + | jq -r --arg k "$ANNOTATION" '.metadata.annotations[$k] // ""') +if [[ "${annotation^^}" != "DRBD,LUKS,STORAGE" ]]; then + echo "FAIL: $RD_DATA annotation $ANNOTATION = '$annotation', want DRBD,LUKS,STORAGE" >&2 + exit 1 +fi + +deadline=$(( $(date +%s) + 30 )) +events=0 +while (( $(date +%s) < deadline )); do + events=$(kubectl get events -A --field-selector "reason=LayerStackNotInherited,involvedObject.name=$RD_DATA" \ + --no-headers 2>/dev/null | wc -l) + (( events > 0 )) && break + sleep 2 +done +if (( events == 0 )); then + echo "FAIL: no LayerStackNotInherited event on $RD_DATA" >&2 + exit 1 +fi + +md5_after=$(read_md5 "$N1" "$DEV" 262144) +if [[ "$md5_before" != "$md5_after" ]]; then + echo "FAIL: $RD_DATA data changed after the group gained LUKS (before=$md5_before after=$md5_after)" >&2 + exit 1 +fi + +wait_layer_stack "$RD_CLONE" "DRBD,STORAGE" + +echo ">> rg-layer-stack-pin OK" diff --git a/tests/operator-harness/lib.sh b/tests/operator-harness/lib.sh index 7f89e50a..a8c00a2c 100755 --- a/tests/operator-harness/lib.sh +++ b/tests/operator-harness/lib.sh @@ -632,6 +632,38 @@ except Exception: print('')" "$vol" 2>/dev/null || echo "") [[ "$actual" == "$expected" ]] ;; + rd_layer_stack) + # The layer stack the RD's CRD carries (spec.layerStack), as an + # upper-cased comma list, equals `expected`. "" means no stack: + # one the controller has yet to decide. Pair with hold_s to prove + # a decided stack is not rewritten (a group change over data). + local rd expected actual + rd=$(substitute "$(python3 -c "import json,sys; print(json.loads(sys.argv[1]).get('rd',''))" "$spec")") + expected=$(python3 -c "import json,sys; print(json.loads(sys.argv[1]).get('expected','').upper())" "$spec") + actual=$(kubectl get resourcedefinition "$rd" -o json 2>/dev/null \ + | python3 -c "import json,sys +try: + print(','.join(json.load(sys.stdin).get('spec', {}).get('layerStack') or []).upper()) +except Exception: + print('')" 2>/dev/null || echo "") + [[ "$actual" == "$expected" ]] + ;; + rd_annotation) + # An annotation on the RD's CRD equals `expected`; "" means the + # key is absent. The CLI shows no annotations, so this reads the + # CRD (blockstor.io/unapplied-group-layer-stack and the like). + local rd key expected actual + rd=$(substitute "$(python3 -c "import json,sys; print(json.loads(sys.argv[1]).get('rd',''))" "$spec")") + key=$(python3 -c "import json,sys; print(json.loads(sys.argv[1]).get('key',''))" "$spec") + expected=$(python3 -c "import json,sys; print(json.loads(sys.argv[1]).get('expected',''))" "$spec") + actual=$(kubectl get resourcedefinition "$rd" -o json 2>/dev/null \ + | python3 -c "import json,sys +try: + print((json.load(sys.stdin).get('metadata', {}).get('annotations') or {}).get(sys.argv[1], '')) +except Exception: + print('')" "$key" 2>/dev/null || echo "") + [[ "$actual" == "$expected" ]] + ;; vd_count) # BUG-048: assert the RD carries EXACTLY `expected` # VolumeDefinitions. A concurrent-auto-assign lost-update diff --git a/tests/operator-harness/replay-runner.sh b/tests/operator-harness/replay-runner.sh index 186020b9..c6191def 100755 --- a/tests/operator-harness/replay-runner.sh +++ b/tests/operator-harness/replay-runner.sh @@ -76,6 +76,11 @@ # `expected` — the /dev/drbd device identity is # stable across a VD-scoped modify (Bug 433); pair # with hold_s (an unset minor reads as "") +# - rd_layer_stack RD.Spec.LayerStack, as an upper-cased comma list, +# equals `expected` ("" = not decided yet); pair +# with hold_s to prove a decided stack holds +# - rd_annotation an annotation on the RD CRD equals `expected` +# ("" = absent) # - vd_count RD carries EXACTLY `expected` VolumeDefinitions # (BUG-048 concurrent-add lost-update catcher) # - pvc_capacity PVC.Status.Capacity.storage matches expected diff --git a/tests/operator-harness/replay/rg-layer-stack-pin.yaml b/tests/operator-harness/replay/rg-layer-stack-pin.yaml new file mode 100644 index 00000000..237e1c1f --- /dev/null +++ b/tests/operator-harness/replay/rg-layer-stack-pin.yaml @@ -0,0 +1,139 @@ +name: rg-layer-stack-pin +description: | + A definition's layer stack is decided once, from its resource group, + and never reshapes a volume that already exists. + + The controller writes the group's stack onto a definition that leaves + it empty. One that holds no data takes the group's stack; one that + already went to disk under the default stack keeps it and is flagged + with the blockstor.io/unapplied-group-layer-stack annotation, so a + LUKS group added over a plaintext volume never has the satellite + luksFormat it. + + Steps: + 1. rg c {{rg}} (no stack) and rg c {{rg}}-luks --layer-list + drbd,luks,storage; rd c {{rd}}-f in {{rg}} stays without a stack; + rd modify --resource-group {{rg}}-luks -> spec.layerStack + DRBD,LUKS,STORAGE. The REST create copies a group's stack itself, + so only a move into the group shows the controller's pin. + 2. rd c + vd c + r c --auto-place=2 in {{rg}} + 3. rg modify {{rg}} --layer-list drbd,luks,storage over that data + 4. rd clone of the data RD straight after: refused (exit 10, "not + pinned yet") while the controller has yet to pin the stack, or + done if it already has; the clone run again succeeds either way + 5. the data RD stays DRBD,STORAGE, held, with the annotation naming + the group's stack; the clone carries DRBD,STORAGE as well + + The LayerStackNotInherited Event and the byte-level check that the + data reads back unchanged live in the L6 cell + tests/e2e/cli-matrix/rg-layer-stack-pin.sh: the runner executes only + `linstor` argv and CRD reads. + + The clone's internal snapshot clone-{{rd}}-c outlives the clone, so it + is deleted before the source in teardown. + +prerequisites: + min_nodes: 2 + storage_pool: stand + +vars: + rd: replay-lspin + sp: stand + +steps: + - name: create-passphrase + # Any Secret-backed passphrase satisfies the LUKS gate; a shared stand + # may already have one (exit 10). + cmd: ["encryption", "create-passphrase", "-p", "replay-lspin-do-not-rely-on"] + expect_exit: [0, 10] + - name: create-plain-group + cmd: ["resource-group", "create", "{{rg}}", "--storage-pool={{sp}}"] + expect_exit: 0 + - name: create-luks-group + cmd: ["resource-group", "create", "{{rg}}-luks", "--layer-list", "drbd,luks,storage", "--storage-pool={{sp}}"] + expect_exit: 0 + - name: rd-in-a-group-without-a-stack-has-none + cmd: ["resource-definition", "create", "--resource-group", "{{rg}}", "{{rd}}-f"] + expect_exit: 0 + await: + kind: rd_layer_stack + rd: "{{rd}}-f" + expected: "" + hold_s: 10 + timeout_s: 30 + - name: rd-moved-into-luks-group-takes-its-stack + # Nothing on the REST side copies the stack on a modify: the + # controller's pin does. + cmd: ["resource-definition", "modify", "--resource-group", "{{rg}}-luks", "{{rd}}-f"] + expect_exit: 0 + await: + kind: rd_layer_stack + rd: "{{rd}}-f" + expected: "DRBD,LUKS,STORAGE" + timeout_s: 60 + + - name: create-rd-with-data + cmd: ["resource-definition", "create", "--resource-group", "{{rg}}", "{{rd}}"] + expect_exit: 0 + - name: create-vd + cmd: ["volume-definition", "create", "{{rd}}", "64M"] + expect_exit: 0 + - name: auto-place-2 + cmd: ["resource", "create", "--auto-place", "2", "--storage-pool={{sp}}", "{{rd}}"] + expect_exit: 0 + await: + kind: all_uptodate + rd: "{{rd}}" + timeout_s: 240 + + - name: group-gains-luks-over-data + cmd: ["resource-group", "modify", "{{rg}}", "--layer-list", "drbd,luks,storage"] + expect_exit: 0 + - name: clone-while-the-stack-may-be-undecided + # 503 "not pinned yet" (exit 10) inside the window before the + # controller's pin, 0 once it has pinned. The resend tolerance is off: + # an "already exists" here would be another fault. + cmd: ["resource-definition", "clone", "{{rd}}", "{{rd}}-c"] + expect_exit: [0, 10] + tolerate_resend_409: false + - name: data-rd-keeps-the-stack-its-data-has + cmd: ["resource-definition", "list", "--resource-definitions", "{{rd}}"] + expect_exit: 0 + await: + kind: rd_layer_stack + rd: "{{rd}}" + expected: "DRBD,STORAGE" + hold_s: 15 + timeout_s: 60 + - name: data-rd-is-flagged-short-of-its-group + cmd: ["resource-definition", "list", "--resource-definitions", "{{rd}}"] + expect_exit: 0 + await: + kind: rd_annotation + rd: "{{rd}}" + key: blockstor.io/unapplied-group-layer-stack + expected: "DRBD,LUKS,STORAGE" + timeout_s: 30 + - name: clone-after-the-pin + # Either the clone of the step before finished and this is its replay, + # or it was refused as undecided and this is the clone itself. + cmd: ["resource-definition", "clone", "{{rd}}", "{{rd}}-c"] + expect_exit: 0 + tolerate_resend_409: false + await: + kind: rd_layer_stack + rd: "{{rd}}-c" + expected: "DRBD,STORAGE" + timeout_s: 60 + +teardown: + - cmd: ["resource-definition", "delete", "{{rd}}-c"] + - cmd: ["snapshot", "delete", "{{rd}}", "clone-{{rd}}-c"] + - cmd: ["resource-definition", "delete", "{{rd}}"] + - cmd: ["resource-definition", "delete", "{{rd}}-f"] + - cmd: ["resource-group", "delete", "{{rg}}"] + - cmd: ["resource-group", "delete", "{{rg}}-luks"] + +invariants: + # Prefix-match on vars.rd also covers the "-c" and "-f" definitions. + - no_orphans From dc7657cba94b70b553c5316a583d137169bcf5fb Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 08:52:40 +0100 Subject: [PATCH 17/22] fix(controller): keep a pinned definition's unapplied-stack mark true to its group The mark recorded the group's stack at pin time and was never looked at again, so a definition whose group later dropped the layer it lacks still read as short of its group. Each reconcile now re-checks a flagged definition: the mark follows the group's current stack, and goes once the group no longer asks for a missing layer or is deleted. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- .../resourcedefinition_layer_stack.go | 83 ++++++++++++++++--- .../resourcedefinition_layer_stack_test.go | 44 ++++++++++ 3 files changed, 116 insertions(+), 13 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index 405a3c97..f0855643 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -32,7 +32,7 @@ The default is `["DRBD","STORAGE"]`, used when neither the RD nor its resource g An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The REST create and `rg spawn` paths copy the group's stack onto the RD when it is created. An RD that ends up with an empty stack in a group that sets one (created with `kubectl apply` or GitOps, or moved by `rd modify --resource-group`) is pinned by the controller, and the satellite builds nothing for it until then: - An RD that holds no data yet gets the group's stack. -- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. +- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. The controller re-checks the annotation each time it reconciles the RD: it follows the group's current stack, and goes once the group no longer asks for a layer the RD lacks or the group is deleted. Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack only when the source set one by choice, and otherwise is left for the controller to pin from its own group. diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index 325e0359..42c2883f 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -58,23 +58,17 @@ const layerStackPinAction = "PinLayerStack" // actually has, the group's stack is recorded in an annotation, and a // warning Event names it, so the volume reads as what it is. func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition) error { - if len(rd.Spec.LayerStack) > 0 || rd.Spec.ResourceGroupName == "" { - return nil + if len(rd.Spec.LayerStack) > 0 { + return r.refreshUnappliedGroupStack(ctx, rd) } - reader := r.directOrCached() - - var rg blockstoriov1alpha1.ResourceGroup - - // The RD keeps the group's name as the operator spelled it; the CR is - // stored under the folded one. - err := reader.Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rg) - if apierrors.IsNotFound(err) { + if rd.Spec.ResourceGroupName == "" { return nil } - if err != nil { - return errors.Wrapf(err, "read resource group %q", rd.Spec.ResourceGroupName) + rg, err := r.readGroup(ctx, rd) + if err != nil || rg == nil { + return err } groupStack := rg.Spec.SelectFilter.LayerStack @@ -116,6 +110,71 @@ func (r *ResourceDefinitionReconciler) pinLayerStack(ctx context.Context, rd *bl return nil } +// readGroup reads the RD's resource group, nil when it does not exist. The +// RD keeps the group's name as the operator spelled it; the CR is stored +// under the folded one. +func (r *ResourceDefinitionReconciler) readGroup( + ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition, +) (*blockstoriov1alpha1.ResourceGroup, error) { + var rg blockstoriov1alpha1.ResourceGroup + + err := r.directOrCached().Get(ctx, client.ObjectKey{Name: k8s.Name(rd.Spec.ResourceGroupName)}, &rg) + if apierrors.IsNotFound(err) { + return nil, nil //nolint:nilnil // no group is an answer, not an error + } + + if err != nil { + return nil, errors.Wrapf(err, "read resource group %q", rd.Spec.ResourceGroupName) + } + + return &rg, nil +} + +// refreshUnappliedGroupStack keeps the annotation on a pinned RD true to its +// group as the group is now: the layers the group asks for while the RD's +// stack lacks one of them, and no annotation once it no longer does or the +// group is gone. An RD without the annotation is left alone; it was never +// short of its group when its stack was decided. +func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( + ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition, +) error { + recorded, flagged := rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] + if !flagged { + return nil + } + + want := "" + + if rd.Spec.ResourceGroupName != "" { + rg, err := r.readGroup(ctx, rd) + if err != nil { + return err + } + + if rg != nil && apiv1.LacksLayerOf(rd.Spec.LayerStack, rg.Spec.SelectFilter.LayerStack) { + want = strings.Join(rg.Spec.SelectFilter.LayerStack, ",") + } + } + + if want == recorded { + return nil + } + + base := rd.DeepCopy() + if want == "" { + delete(rd.Annotations, apiv1.UnappliedGroupLayerStackAnnotation) + } else { + rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] = want + } + + err := r.Patch(ctx, rd, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) + if err != nil { + return errors.Wrapf(err, "refresh the unapplied group stack of %q", rd.Name) + } + + return nil +} + // stackToPin is the group's stack for an RD with nothing on disk, and // the default stack, the one an empty stack was built as, for an RD // that already holds data. diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go index f16b48c1..1ea80146 100644 --- a/internal/controller/resourcedefinition_layer_stack_test.go +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -425,3 +425,47 @@ func TestPinLayerStackTreatsTheInitializedLatchAsData(t *testing.T) { t.Fatalf("stack %v, want the applied default", got.rd.Spec.LayerStack) } } + +// The annotation on a pinned RD follows its group: it goes once the group +// no longer asks for a layer the RD lacks, or the group is gone, and names +// the group's current stack while the RD is still short of it. An RD that +// was never flagged is left alone. +func TestPinnedRDsUnappliedGroupStackFollowsTheGroup(t *testing.T) { + t.Parallel() + + flagged := func(value string) *blockstoriov1alpha1.ResourceDefinition { + rd := pinRD("rg-pin", "DRBD", "STORAGE") + rd.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: value} + + return rd + } + + for name, tc := range map[string]struct { + objs []client.Object + want string + }{ + "group dropped the layer": {objs: []client.Object{pinGroup("DRBD", "STORAGE"), flagged("DRBD,LUKS,STORAGE")}}, + "group gone": {objs: []client.Object{flagged("DRBD,LUKS,STORAGE")}}, + "group still asks": {objs: []client.Object{pinGroup(luksStack...), flagged("DRBD,LUKS,STORAGE")}, want: "DRBD,LUKS,STORAGE"}, + "group changed its ask": {objs: []client.Object{pinGroup("LUKS", "STORAGE"), flagged("DRBD,LUKS,STORAGE")}, want: "LUKS,STORAGE"}, + "never flagged": {objs: []client.Object{pinGroup(luksStack...), pinRD("rg-pin", "DRBD", "STORAGE")}}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + got := reconcilePin(t, pinClient(t, tc.objs...)) + if got.err != nil { + t.Fatalf("reconcile: %v", got.err) + } + + a, ok := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] + if a != tc.want || ok != (tc.want != "") { + t.Errorf("annotation %q (present %v), want %q", a, ok, tc.want) + } + + if !slices.Equal(got.rd.Spec.LayerStack, []string{"DRBD", "STORAGE"}) { + t.Errorf("the pinned stack changed to %v", got.rd.Spec.LayerStack) + } + }) + } +} From f30b4cea97ae3695c5a69b49ff91f11ab4da17e5 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 08:52:40 +0100 Subject: [PATCH 18/22] fix(luks): warn against wiping a volume's own metadata in the format refusal The hint suggested wipefs for any signature, including DRBD metadata on a volume that is already in use, where wiping destroys the volume. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/luks/luks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/luks/luks.go b/pkg/luks/luks.go index 45c5afff..98fd3b0f 100644 --- a/pkg/luks/luks.go +++ b/pkg/luks/luks.go @@ -90,7 +90,7 @@ func (c *Cryptsetup) Format(ctx context.Context, device string, key []byte) erro if len(signatures) > 0 { return errors.Wrapf(ErrDeviceHoldsData, - "%s carries %s; refusing luksFormat. If the volume is new and the signature is left over from an earlier one on the same extents, clear it with wipefs", + "%s carries %s; refusing luksFormat. Only if nothing was ever written to this volume and the signature is left over from an earlier one on the same extents, clear it with wipefs; DRBD metadata on a volume already in use is its own", device, strings.Join(signatures, ", ")) } From ae80f231bfcafa785599dcd2e25b92a8125218a5 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 09:27:28 +0100 Subject: [PATCH 19/22] fix(controller): flag every decided stack short of its group, both ways The refresh looked only at a definition that already carried the mark, and could now remove it, so a group that dropped a layer and took it back left the definition unflagged for good. Every definition with a decided stack is now checked against its group's current stack: flagged, with a warning Event, while it lacks a layer of it, and cleared once it does not. That also flags one whose group gains a layer after the pin. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- .../resourcedefinition_layer_stack.go | 32 ++++++---- .../resourcedefinition_layer_stack_test.go | 59 ++++++++++++++++--- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index f0855643..bf5bf3ed 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -32,7 +32,7 @@ The default is `["DRBD","STORAGE"]`, used when neither the RD nor its resource g An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The REST create and `rg spawn` paths copy the group's stack onto the RD when it is created. An RD that ends up with an empty stack in a group that sets one (created with `kubectl apply` or GitOps, or moved by `rd modify --resource-group`) is pinned by the controller, and the satellite builds nothing for it until then: - An RD that holds no data yet gets the group's stack. -- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. The controller re-checks the annotation each time it reconciles the RD: it follows the group's current stack, and goes once the group no longer asks for a layer the RD lacks or the group is deleted. +- An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. The controller re-checks every RD with a decided stack each time it reconciles it: the annotation names the group's current stack whenever the RD's stack lacks a layer of it, set by hand or pinned, and goes once the group no longer asks for a missing layer or is deleted. A newly flagged RD gets the same warning Event. Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack only when the source set one by choice, and otherwise is left for the controller to pin from its own group. diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index 42c2883f..6418740f 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -130,19 +130,17 @@ func (r *ResourceDefinitionReconciler) readGroup( return &rg, nil } -// refreshUnappliedGroupStack keeps the annotation on a pinned RD true to its -// group as the group is now: the layers the group asks for while the RD's -// stack lacks one of them, and no annotation once it no longer does or the -// group is gone. An RD without the annotation is left alone; it was never -// short of its group when its stack was decided. +// refreshUnappliedGroupStack keeps the annotation on an RD with a decided +// stack true to its group as the group is now, both ways: it names the +// group's stack while the RD's stack lacks a layer of it, and is gone once +// it no longer does or the group is deleted. So an RD whose group gains a +// layer after the pin, or a copy whose group asks for more than its source +// had, is flagged too, and a group that drops a layer and takes it back +// flags the RD again. The group is read from the controller's cache. func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition, ) error { - recorded, flagged := rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] - if !flagged { - return nil - } - + recorded := rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] want := "" if rd.Spec.ResourceGroupName != "" { @@ -161,9 +159,13 @@ func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( } base := rd.DeepCopy() - if want == "" { + + switch { + case want == "": delete(rd.Annotations, apiv1.UnappliedGroupLayerStackAnnotation) - } else { + case rd.Annotations == nil: + rd.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: want} + default: rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] = want } @@ -172,6 +174,12 @@ func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( return errors.Wrapf(err, "refresh the unapplied group stack of %q", rd.Name) } + if recorded == "" && r.Recorder != nil { + r.Recorder.Eventf(rd, nil, corev1.EventTypeWarning, layerStackNotInheritedReason, layerStackPinAction, + "layer stack %s lacks a layer of the stack %s its resource group %s now asks for; migrate the data to a new volume to take it", + strings.Join(rd.Spec.LayerStack, ","), want, rd.Spec.ResourceGroupName) + } + return nil } diff --git a/internal/controller/resourcedefinition_layer_stack_test.go b/internal/controller/resourcedefinition_layer_stack_test.go index 1ea80146..e042e52b 100644 --- a/internal/controller/resourcedefinition_layer_stack_test.go +++ b/internal/controller/resourcedefinition_layer_stack_test.go @@ -283,12 +283,14 @@ func TestPinLayerStackLooksOnlyAtItsOwnReplicas(t *testing.T) { } } -// Every shape that has nothing to inherit is left exactly as it is. +// Every shape that has nothing to inherit is left exactly as it is. An RD +// that set its own stack short of its group is flagged instead, see +// TestPinnedRDsUnappliedGroupStackFollowsTheGroup. func TestPinLayerStackLeavesRDsWithNothingToInherit(t *testing.T) { t.Parallel() cases := map[string][]client.Object{ - "RD sets its own stack": {pinGroup(luksStack...), pinRD("rg-pin", "DRBD", "STORAGE")}, + "RD sets its own stack": {pinGroup(luksStack...), pinRD("rg-pin", luksStack...)}, "RD in no group": {pinGroup(luksStack...), pinRD("")}, "group gone": {pinRD("rg-pin")}, "group sets no stack": {pinGroup(), pinRD("rg-pin")}, @@ -426,10 +428,10 @@ func TestPinLayerStackTreatsTheInitializedLatchAsData(t *testing.T) { } } -// The annotation on a pinned RD follows its group: it goes once the group -// no longer asks for a layer the RD lacks, or the group is gone, and names -// the group's current stack while the RD is still short of it. An RD that -// was never flagged is left alone. +// The annotation on a pinned RD follows its group both ways: it goes once +// the group no longer asks for a layer the RD lacks, or the group is gone, +// and names the group's current stack while the RD is short of it, whether +// or not it was flagged before. func TestPinnedRDsUnappliedGroupStackFollowsTheGroup(t *testing.T) { t.Parallel() @@ -448,7 +450,8 @@ func TestPinnedRDsUnappliedGroupStackFollowsTheGroup(t *testing.T) { "group gone": {objs: []client.Object{flagged("DRBD,LUKS,STORAGE")}}, "group still asks": {objs: []client.Object{pinGroup(luksStack...), flagged("DRBD,LUKS,STORAGE")}, want: "DRBD,LUKS,STORAGE"}, "group changed its ask": {objs: []client.Object{pinGroup("LUKS", "STORAGE"), flagged("DRBD,LUKS,STORAGE")}, want: "LUKS,STORAGE"}, - "never flagged": {objs: []client.Object{pinGroup(luksStack...), pinRD("rg-pin", "DRBD", "STORAGE")}}, + "group gained a layer": {objs: []client.Object{pinGroup(luksStack...), pinRD("rg-pin", "DRBD", "STORAGE")}, want: "DRBD,LUKS,STORAGE"}, + "never short": {objs: []client.Object{pinGroup("DRBD", "STORAGE"), pinRD("rg-pin", "DRBD", "STORAGE")}}, } { t.Run(name, func(t *testing.T) { t.Parallel() @@ -469,3 +472,45 @@ func TestPinnedRDsUnappliedGroupStackFollowsTheGroup(t *testing.T) { }) } } + +// A group that drops a layer and takes it back flags its RD again, with a +// warning Event: the mark is not lost to the flap. +func TestUnappliedGroupStackSurvivesAGroupFlap(t *testing.T) { + t.Parallel() + + rd := pinRD("rg-pin", "DRBD", "STORAGE") + rd.Annotations = map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: "DRBD,LUKS,STORAGE"} + group := pinGroup(luksStack...) + cli := pinClient(t, group, rd) + + setGroupStack := func(stack ...string) { + t.Helper() + + var live blockstoriov1alpha1.ResourceGroup + if err := cli.Get(context.Background(), client.ObjectKey{Name: "rg-pin"}, &live); err != nil { + t.Fatalf("get group: %v", err) + } + + live.Spec.SelectFilter.LayerStack = stack + if err := cli.Update(context.Background(), &live); err != nil { + t.Fatalf("update group: %v", err) + } + } + + setGroupStack("DRBD", "STORAGE") + + if got := reconcilePin(t, cli); got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] != "" { + t.Fatalf("still flagged after the group dropped LUKS: %v", got.rd.Annotations) + } + + setGroupStack(luksStack...) + + got := reconcilePin(t, cli) + if a := got.rd.Annotations[apiv1.UnappliedGroupLayerStackAnnotation]; a != "DRBD,LUKS,STORAGE" { + t.Fatalf("annotation %q after the group took LUKS back, want the group's stack", a) + } + + if len(got.events) != 1 || !strings.HasPrefix(got.events[0], "Warning LayerStackNotInherited") { + t.Errorf("events %v, want one LayerStackNotInherited warning", got.events) + } +} From 3b6a285a578f9c2b325b5528def70b409dcda2cd Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 09:28:31 +0100 Subject: [PATCH 20/22] fix(lvm): replace a temporary thick LV a crash left instead of reusing it The next create renamed the leftover whatever its size, so a volume grown in between came up at the size the crashed call asked for. The leftover was never exposed under the volume's name, so it is removed and the LV is created again at the volume's current size. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- pkg/storage/lvm/lvm_thick.go | 42 +++++++++++++++++------------- pkg/storage/lvm/thick_wipe_test.go | 17 ++++++------ 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/pkg/storage/lvm/lvm_thick.go b/pkg/storage/lvm/lvm_thick.go index 773aa2b5..c90ee598 100644 --- a/pkg/storage/lvm/lvm_thick.go +++ b/pkg/storage/lvm/lvm_thick.go @@ -73,8 +73,9 @@ func (*Thick) ResizeZeroFills() bool { return false } // created under a temporary name (freshLVName), wiped there, and only // then renamed to the volume's name: nothing ever finds an unwiped LV // under a name it treats as a volume, this satellite after a crash or an -// older one after a rollback. A temporary LV left by a crash is wiped -// and renamed by the next call; it was never exposed. +// older one after a rollback. A temporary LV left by a crash was never +// exposed, so the next call removes it and creates the LV again, at the +// size the volume has now rather than whatever the crashed call asked. func (t *Thick) CreateVolume(ctx context.Context, vol storage.Volume) error { final := volumeLVName(vol) fresh := freshLVName(final) @@ -88,26 +89,31 @@ func (t *Thick) CreateVolume(ctx context.Context, vol storage.Volume) error { return nil } - if !names[fresh] { - sizeMiB := max(vol.SizeKib/mibPerKib, 1) - - // Skip the optional zero-on-create step (-Wn -Zn) — the satellite - // container has no udev daemon, and the wipe trips on the missing - // /dev// symlink. Same trick as install-pools.sh. Bug 305: - // the udev-less activation section rides in the SAME single - // `--config` as the device filter (ArgsUdevless) — a separate - // `--config activation{…}` would be a repeated `--config`, which - // the LVM CLI rejects. - _, err = t.exec.Run(ctx, "lvcreate", - ArgsUdevless("--size", strconv.FormatInt(sizeMiB, 10)+"MiB", - "--name", fresh, - "-Wn", "-Zn", - t.cfg.VolumeGroup)...) + if names[fresh] { + _, err = t.exec.Run(ctx, "lvremove", Args("--force", t.cfg.VolumeGroup+"/"+fresh)...) if err != nil { - return errors.Wrapf(err, "lvcreate %s", fresh) + return errors.Wrapf(err, "lvremove the leftover %s", fresh) } } + sizeMiB := max(vol.SizeKib/mibPerKib, 1) + + // Skip the optional zero-on-create step (-Wn -Zn) — the satellite + // container has no udev daemon, and the wipe trips on the missing + // /dev// symlink. Same trick as install-pools.sh. Bug 305: + // the udev-less activation section rides in the SAME single + // `--config` as the device filter (ArgsUdevless) — a separate + // `--config activation{…}` would be a repeated `--config`, which + // the LVM CLI rejects. + _, err = t.exec.Run(ctx, "lvcreate", + ArgsUdevless("--size", strconv.FormatInt(sizeMiB, 10)+"MiB", + "--name", fresh, + "-Wn", "-Zn", + t.cfg.VolumeGroup)...) + if err != nil { + return errors.Wrapf(err, "lvcreate %s", fresh) + } + device := "/dev/" + t.cfg.VolumeGroup + "/" + fresh _, err = t.exec.Run(ctx, "wipefs", "--all", "--force", device) diff --git a/pkg/storage/lvm/thick_wipe_test.go b/pkg/storage/lvm/thick_wipe_test.go index 1374cacd..5b569eb1 100644 --- a/pkg/storage/lvm/thick_wipe_test.go +++ b/pkg/storage/lvm/thick_wipe_test.go @@ -77,9 +77,10 @@ func TestThickCreateVolumeLeavesAnLVUnderTheVolumesNameAlone(t *testing.T) { } // A crash between the create and the rename leaves the temporary LV. It was -// never exposed under the volume's name, so the next call wipes it and -// renames it rather than creating another. -func TestThickCreateVolumeFinishesATemporaryLVACrashLeft(t *testing.T) { +// never exposed under the volume's name, so the next call removes it and +// creates the LV again at the volume's current size, which the leftover +// need not have. +func TestThickCreateVolumeReplacesATemporaryLVACrashLeft(t *testing.T) { fx := storage.NewFakeExec() fx.Expect(lvsVG, storage.FakeResponse{Stdout: []byte(" pvc-1_00000.bsnew\n")}) @@ -88,13 +89,11 @@ func TestThickCreateVolumeFinishesATemporaryLVACrashLeft(t *testing.T) { } lines := fx.CommandLines() - if slices.Contains(lines, lvcreateLV) { - t.Errorf("created a second LV over the leftover: %v", lines) - } + rm := slices.Index(lines, "lvremove --config devices { filter=['r|^/dev/drbd|','r|^/dev/zd|'] } --force vg/pvc-1_00000.bsnew") + c, w, r := slices.Index(lines, lvcreateLV), slices.Index(lines, wipeFresh), slices.Index(lines, renameLV) - w, r := slices.Index(lines, wipeFresh), slices.Index(lines, renameLV) - if w < 0 || r < 0 || w >= r { - t.Errorf("want the leftover wiped, then renamed; got %v", lines) + if rm < 0 || c < rm || w < c || r < w { + t.Errorf("want the leftover removed, then a new LV created, wiped and renamed; got %v", lines) } } From 67d8bdcee2ac46341281148594eb3a44ad0ab061 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 10:45:31 +0100 Subject: [PATCH 21/22] fix(rest): keep a hand-set stack on a volume-less clone of a flagged source Since the mark also flags a stack set by hand that lacks a layer of the group, a volume-less clone dropped such a stack, LUKS,STORAGE in a DRBD,STORAGE group say, and the controller pinned the clone to the group's plain stack. A stack pinned over data is always the default one, so only a flagged source with the default stack hands the clone nothing now. The annotation's comment and the docs describe its wider meaning. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- docs/layer-stack.md | 2 +- pkg/api/v1/annotations.go | 14 +++++++----- pkg/rest/rd_clone.go | 20 ++++++++++++----- pkg/rest/restore_effective_layers_test.go | 27 +++++++++++++++++++++++ 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/docs/layer-stack.md b/docs/layer-stack.md index bf5bf3ed..81432446 100644 --- a/docs/layer-stack.md +++ b/docs/layer-stack.md @@ -34,7 +34,7 @@ An RD's stack is fixed once and then read only from `RD.Spec.LayerStack`. The RE - An RD that holds no data yet gets the group's stack. - An RD that already holds data (a replica reported DRBD metadata or a device, or `spec.initialized` is set) keeps `["DRBD","STORAGE"]`, the stack its storage was built with. When that is missing a layer the group asks for, the group's stack is recorded in the `blockstor.io/unapplied-group-layer-stack` annotation and a `LayerStackNotInherited` warning Event is emitted on the RD. Such a volume is not what its group promises (for example, a plaintext volume in a LUKS group); move its data to a new volume to get the group's stack. The controller re-checks every RD with a decided stack each time it reconciles it: the annotation names the group's current stack whenever the RD's stack lacks a layer of it, set by hand or pinned, and goes once the group no longer asks for a missing layer or is deleted. A newly flagged RD gets the same warning Event. -Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack only when the source set one by choice, and otherwise is left for the controller to pin from its own group. +Clones and restores copy bytes, so the RD they create takes the stack those bytes were written under: the source's own, or the default when neither the source nor its group sets one. A source still waiting for the controller to pin its stack is answered 503 (the CLI exits with an error) until it is pinned, and a prepared restore target in the same state is refused the same way before anything is written to it. A copy whose stack is short of what its own group asks for carries the `blockstor.io/unapplied-group-layer-stack` annotation. A clone of a volume-less RD copies no bytes: it takes the source's stack, unless that is the default stack and the source carries the annotation, which is how a stack pinned over data looks. Such a clone, and one of a source with no stack, is left for the controller to pin from its own group. Changing a group's stack later never reshapes RDs that already carry one, as in upstream LINSTOR, where the stack is decided when a resource is created. diff --git a/pkg/api/v1/annotations.go b/pkg/api/v1/annotations.go index 4fe3181f..0af5d742 100644 --- a/pkg/api/v1/annotations.go +++ b/pkg/api/v1/annotations.go @@ -142,10 +142,12 @@ const PeerChangedAnnotation = "blockstor.io/peer-changed" const RDSpawnShortfallAnnotation = "blockstor.io/spawn-shortfall" // UnappliedGroupLayerStackAnnotation is stamped on a ResourceDefinition -// whose data was written under a stack missing a layer its resource group -// asks for: one the controller pinned over data it found under the default -// stack, or a clone or restore of such bytes. It records, comma-separated, -// the group's stack, so an operator can find every volume that is short of -// what its group promises (for example a plaintext volume in a LUKS group) -// and migrate it. +// whose decided stack lacks a layer its resource group asks for now: one the +// controller pinned to the default stack over data it found there, a clone +// or restore of such bytes, a stack the operator set by hand, or a stack the +// group outgrew after it was decided. The controller keeps it true to the +// group's current stack on every reconcile and clears it once nothing is +// missing. It records, comma-separated, the group's stack, so an operator +// can find every volume that is short of what its group promises (for +// example a plaintext volume in a LUKS group). const UnappliedGroupLayerStackAnnotation = "blockstor.io/unapplied-group-layer-stack" diff --git a/pkg/rest/rd_clone.go b/pkg/rest/rd_clone.go index 5eb272b3..c66e9214 100644 --- a/pkg/rest/rd_clone.go +++ b/pkg/rest/rd_clone.go @@ -348,22 +348,32 @@ func withoutUnappliedGroupStack(annotations map[string]string) map[string]string // caller's, or else the source's as stored. A shell holds no bytes, so nothing // forces a stack on it the way the data path is forced; left empty, it takes // its own group's stack from the controller, which pins it once -// (pinLayerStack), exactly like a definition created empty. A source whose -// stack was pinned over data that was already there, which the -// unapplied-group annotation marks, passes that stack on to no shell: it was -// the data's, not a choice. +// (pinLayerStack), exactly like a definition created empty. +// +// A source whose stack the controller pinned over data that was already +// there passes that stack on to no shell: it was the data's, not a choice. +// The unapplied-group annotation alone does not say that, since it also +// marks a stack set by hand that lacks a layer of the group; but a stack +// pinned over data is always the default one. So only a flagged source +// with the default stack hands the shell nothing, and a flagged stack of +// the operator's own, LUKS,STORAGE in a DRBD,STORAGE group say, is copied. func cloneShellLayerStack(src *apiv1.ResourceDefinition, req *rdCloneRequest) []string { if len(req.LayerList) > 0 { return req.LayerList } - if src.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] != "" { + if src.Annotations[apiv1.UnappliedGroupLayerStackAnnotation] != "" && isDefaultLayerStack(src.LayerStack) { return nil } return slices.Clone(src.LayerStack) } +// isDefaultLayerStack reports whether stack is the default one, in any case. +func isDefaultLayerStack(stack []string) bool { + return slices.EqualFunc(stack, apiv1.DefaultLayerStack(), strings.EqualFold) +} + // clonePassphraseHolds refuses a clone whose stack carries LUKS while the // cluster passphrase is missing. False means a refusal has been written. // diff --git a/pkg/rest/restore_effective_layers_test.go b/pkg/rest/restore_effective_layers_test.go index f097dbe7..513b244f 100644 --- a/pkg/rest/restore_effective_layers_test.go +++ b/pkg/rest/restore_effective_layers_test.go @@ -252,6 +252,33 @@ func TestRDCloneShellDoesNotInheritAStackPinnedOverData(t *testing.T) { assertUnapplied(t, st, "src-pinned", "DRBD,LUKS,STORAGE") } +// A stack the operator set by hand is copied onto the shell even when the +// controller flagged it as short of the group: a flag on anything but the +// default stack cannot be a pin over data, so the clone keeps its LUKS. +func TestRDCloneShellKeepsAFlaggedStackTheSourceChose(t *testing.T) { + t.Parallel() + + st := store.NewInMemory() + seedLayeredGroup(t, st, "grp-plain", apiv1.DefaultLayerStack()) + + chosen := []string{apiv1.LayerKindLUKS, apiv1.LayerKindStorage} + if err := st.ResourceDefinitions().Create(t.Context(), &apiv1.ResourceDefinition{ + Name: "src-own", ResourceGroupName: "grp-plain", LayerStack: chosen, + Annotations: map[string]string{apiv1.UnappliedGroupLayerStackAnnotation: "DRBD,STORAGE"}, + }); err != nil { + t.Fatalf("seed the volume-less source: %v", err) + } + + base, stop := startServerWithPassphrase(t, st) + defer stop() + + if code := cloneOnce(t, base, "src-own", "dst-own", nil); code != http.StatusCreated { + t.Fatalf("shell clone = %d, want 201", code) + } + + assertStack(t, st, "dst-own", chosen) +} + // A stack the source chose is copied onto its shell. func TestRDCloneShellKeepsAStackTheSourceChose(t *testing.T) { t.Parallel() From 04acafe0ed411b632a1c7a02077735d6d8a1f0ca Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 26 Oct 2026 10:45:31 +0100 Subject: [PATCH 22/22] fix(controller): re-check a group's definitions when its layer stack changes The mark caught up with a group edit only when the definition was reconciled for another reason or at the resync. A watch on groups now enqueues the definitions in one whose stack changed. The refresh's Event no longer tells a definition that holds nothing to migrate its data, and its comment says the group is read directly. Assisted-by: LLM Signed-off-by: Andrei Kvapil --- .../group_layer_stack_watch_test.go | 79 +++++++++++++++++++ .../resourcedefinition_controller.go | 5 ++ .../resourcedefinition_layer_stack.go | 50 +++++++++++- 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 internal/controller/group_layer_stack_watch_test.go diff --git a/internal/controller/group_layer_stack_watch_test.go b/internal/controller/group_layer_stack_watch_test.go new file mode 100644 index 00000000..589d9012 --- /dev/null +++ b/internal/controller/group_layer_stack_watch_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +/* +Copyright 2026 Cozystack contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/event" + + blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" +) + +func watchGroup(stack ...string) *blockstoriov1alpha1.ResourceGroup { + return &blockstoriov1alpha1.ResourceGroup{ + ObjectMeta: metav1.ObjectMeta{Name: "rg-watch"}, + Spec: blockstoriov1alpha1.ResourceGroupSpec{ + SelectFilter: blockstoriov1alpha1.ResourceGroupSelectFilter{LayerStack: stack}, + }, + } +} + +// Only a change to a group's layer stack re-checks its RDs. +func TestGroupLayerStackChangedFiresOnlyOnAStackChange(t *testing.T) { + t.Parallel() + + pred := groupLayerStackChanged() + + if !pred.Update(event.UpdateEvent{ObjectOld: watchGroup("DRBD", "STORAGE"), ObjectNew: watchGroup("DRBD", "LUKS", "STORAGE")}) { + t.Error("a group that gained LUKS did not re-check its RDs") + } + + if pred.Update(event.UpdateEvent{ObjectOld: watchGroup("DRBD", "STORAGE"), ObjectNew: watchGroup("DRBD", "STORAGE")}) { + t.Error("a group update that left the stack alone re-checked its RDs") + } +} + +// The group's RDs are enqueued under any spelling of its name, and no other. +func TestEnqueueRDsInGroupFindsItsRDsOnly(t *testing.T) { + t.Parallel() + + rd := func(name, group string) *blockstoriov1alpha1.ResourceDefinition { + return &blockstoriov1alpha1.ResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: blockstoriov1alpha1.ResourceDefinitionSpec{ResourceGroupName: group}, + } + } + + cli := fake.NewClientBuilder().WithScheme(bug048Scheme(t)). + WithObjects(rd("pvc-a", "rg-watch"), rd("pvc-b", "RG-Watch"), rd("pvc-c", "rg-other"), rd("pvc-d", "")).Build() + r := &ResourceDefinitionReconciler{Client: cli} + + got := map[string]bool{} + for _, req := range r.enqueueRDsInGroup(context.Background(), watchGroup()) { + got[req.Name] = true + } + + if len(got) != 2 || !got["pvc-a"] || !got["pvc-b"] { + t.Errorf("enqueued %v, want pvc-a and pvc-b", got) + } +} diff --git a/internal/controller/resourcedefinition_controller.go b/internal/controller/resourcedefinition_controller.go index a6a82f29..a501ed7d 100644 --- a/internal/controller/resourcedefinition_controller.go +++ b/internal/controller/resourcedefinition_controller.go @@ -1795,6 +1795,11 @@ func (r *ResourceDefinitionReconciler) SetupWithManager(mgr ctrl.Manager) error Watches(&blockstoriov1alpha1.Node{}, handler.EnqueueRequestsFromMapFunc(r.enqueueRDsForNode), builder.WithPredicates(nodeDrainFlagChanged())). + // A group that gains or drops a layer re-checks its RDs' + // unapplied-group annotation now, not at the next resync. + Watches(&blockstoriov1alpha1.ResourceGroup{}, + handler.EnqueueRequestsFromMapFunc(r.enqueueRDsInGroup), + builder.WithPredicates(groupLayerStackChanged())). Named("resourcedefinition"). Complete(r) } diff --git a/internal/controller/resourcedefinition_layer_stack.go b/internal/controller/resourcedefinition_layer_stack.go index 6418740f..a1423ed4 100644 --- a/internal/controller/resourcedefinition_layer_stack.go +++ b/internal/controller/resourcedefinition_layer_stack.go @@ -27,8 +27,12 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" blockstoriov1alpha1 "github.com/cozystack/blockstor/api/v1alpha1" apiv1 "github.com/cozystack/blockstor/pkg/api/v1" @@ -136,7 +140,8 @@ func (r *ResourceDefinitionReconciler) readGroup( // it no longer does or the group is deleted. So an RD whose group gains a // layer after the pin, or a copy whose group asks for more than its source // had, is flagged too, and a group that drops a layer and takes it back -// flags the RD again. The group is read from the controller's cache. +// flags the RD again. The group is read with the reconciler's reader, a +// direct GET when the manager wires the APIReader, as it does in production. func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( ctx context.Context, rd *blockstoriov1alpha1.ResourceDefinition, ) error { @@ -176,7 +181,7 @@ func (r *ResourceDefinitionReconciler) refreshUnappliedGroupStack( if recorded == "" && r.Recorder != nil { r.Recorder.Eventf(rd, nil, corev1.EventTypeWarning, layerStackNotInheritedReason, layerStackPinAction, - "layer stack %s lacks a layer of the stack %s its resource group %s now asks for; migrate the data to a new volume to take it", + "layer stack %s lacks a layer of the stack %s its resource group %s asks for", strings.Join(rd.Spec.LayerStack, ","), want, rd.Spec.ResourceGroupName) } @@ -237,3 +242,44 @@ func (r *ResourceDefinitionReconciler) rdHoldsData(ctx context.Context, rdName s return false, nil } + +// groupLayerStackChanged fires on a group update that changes its layer +// stack, the only group edit the unapplied-group annotation depends on. +// Creates and deletes are left to the RD's own reconcile: a new group has +// no pinned RDs yet, and a deleted one is read as gone there. +func groupLayerStackChanged() predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + DeleteFunc: func(event.DeleteEvent) bool { return false }, + GenericFunc: func(event.GenericEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldGroup, okOld := e.ObjectOld.(*blockstoriov1alpha1.ResourceGroup) + newGroup, okNew := e.ObjectNew.(*blockstoriov1alpha1.ResourceGroup) + + return okOld && okNew && + !slices.Equal(oldGroup.Spec.SelectFilter.LayerStack, newGroup.Spec.SelectFilter.LayerStack) + }, + } +} + +// enqueueRDsInGroup maps a group event to the RDs that name it, under any +// spelling of the name the store folds to the group's. +func (r *ResourceDefinitionReconciler) enqueueRDsInGroup(ctx context.Context, obj client.Object) []reconcile.Request { + var rdList blockstoriov1alpha1.ResourceDefinitionList + + err := r.List(ctx, &rdList) + if err != nil { + return nil + } + + var out []reconcile.Request + + for i := range rdList.Items { + group := rdList.Items[i].Spec.ResourceGroupName + if group != "" && k8s.Name(group) == obj.GetName() { + out = append(out, reconcile.Request{NamespacedName: types.NamespacedName{Name: rdList.Items[i].Name}}) + } + } + + return out +}