From 6cb85cec059e11ed103cf6dced49f5b962d4f2da Mon Sep 17 00:00:00 2001 From: Andrii Chubatiuk Date: Mon, 22 Jun 2026 12:17:41 +0300 Subject: [PATCH] vmcluster: support storage node pools --- api/operator/v1/vlcluster_types.go | 16 +- api/operator/v1/vtcluster_types.go | 16 +- api/operator/v1alpha1/vmdistributed_types.go | 6 +- .../v1alpha1/vmdistributed_types_test.go | 25 +- api/operator/v1beta1/vmcluster_types.go | 484 +++++- api/operator/v1beta1/vmcluster_types_test.go | 586 +++++-- api/operator/v1beta1/vmextra_types.go | 73 + api/operator/v1beta1/vmextra_types_test.go | 182 ++ api/operator/v1beta1/vmuser_types.go | 6 +- api/operator/v1beta1/zz_generated.deepcopy.go | 64 +- config/crd/overlay/crd.descriptionless.yaml | 55 + config/crd/overlay/crd.yaml | 137 ++ docs/CHANGELOG.md | 2 + docs/api.md | 68 +- docs/resources/vmcluster.md | 151 ++ .../operator/factory/build/build.go | 66 - .../operator/factory/build/build_test.go | 147 -- .../operator/factory/build/cluster.go | 26 +- .../operator/factory/build/defaults.go | 90 +- .../operator/factory/build/defaults_test.go | 60 +- .../operator/factory/build/security.go | 30 +- .../factory/reconcile/vmcluster_test.go | 4 +- .../operator/factory/vmauth/vmusers_config.go | 125 +- .../factory/vmauth/vmusers_config_test.go | 128 +- .../operator/factory/vmcluster/vmcluster.go | 549 +++--- .../factory/vmcluster/vmcluster_pools.go | 145 ++ .../vmcluster/vmcluster_reconcile_test.go | 121 +- .../factory/vmcluster/vmcluster_test.go | 1544 ++++++++++++----- .../factory/vmdistributed/util_test.go | 56 +- .../vmdistributed_reconcile_test.go | 12 +- .../vmdistributed/vmdistributed_test.go | 28 +- .../operator/factory/vmdistributed/zone.go | 2 +- .../factory/vmdistributed/zone_test.go | 12 +- .../reconcile_and_track_status_test.go | 4 +- internal/converter/converter.go | 21 +- internal/converter/converter_test.go | 13 + internal/podutil/util.go | 4 +- .../operator/v1beta1/vmcluster_webhook.go | 41 +- test/e2e/upgrade/upgrade_test.go | 6 +- test/e2e/utils_test.go | 17 +- test/e2e/vmcluster_test.go | 1130 +++++++----- test/e2e/vmdistributed_test.go | 58 +- test/e2e/watchnamespace/controllers_test.go | 4 +- 43 files changed, 4474 insertions(+), 1840 deletions(-) create mode 100644 internal/controller/operator/factory/vmcluster/vmcluster_pools.go diff --git a/api/operator/v1/vlcluster_types.go b/api/operator/v1/vlcluster_types.go index c6d3ea4699..d2eeb52008 100644 --- a/api/operator/v1/vlcluster_types.go +++ b/api/operator/v1/vlcluster_types.go @@ -875,14 +875,14 @@ func (cr *VLCluster) IsOwnsServiceAccount() bool { // AsURL implements stub for interface. // nolint:dupl,lll -func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string { +func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) (string, error) { var defaultPort string var svcSpec *vmv1beta1.AdditionalServiceSpec var extraArgs map[string]string switch kind { case vmv1beta1.ClusterComponentSelect: if cr.Spec.VLSelect == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlSelect configured", cr.Name) } defaultPort = "9471" if cr.Spec.VLSelect.Port != "" { @@ -892,7 +892,7 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.VLSelect.ExtraArgs case vmv1beta1.ClusterComponentInsert: if cr.Spec.VLInsert == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlInsert configured", cr.Name) } defaultPort = "9481" if cr.Spec.VLInsert.Port != "" { @@ -902,7 +902,7 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.VLInsert.ExtraArgs case vmv1beta1.ClusterComponentStorage: if cr.Spec.VLStorage == nil { - return "" + return "", fmt.Errorf("vlcluster %q has no spec.vlStorage configured", cr.Name) } defaultPort = "9491" if cr.Spec.VLStorage.Port != "" { @@ -914,12 +914,16 @@ func (cr *VLCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string panic("BUG unsupported cluster kind=" + string(kind)) } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port), nil } // GetRemoteWriteURL returns the insert URL for VLCluster (used by VLDistributed) func (cr *VLCluster) GetRemoteWriteURL() string { - return cr.AsURL(vmv1beta1.ClusterComponentInsert, false) + "/insert/native" + url, err := cr.AsURL(vmv1beta1.ClusterComponentInsert, false) + if err != nil { + return "" + } + return url + "/insert/native" } // +kubebuilder:object:root=true diff --git a/api/operator/v1/vtcluster_types.go b/api/operator/v1/vtcluster_types.go index 5b07ff25fc..7c7b0b526e 100644 --- a/api/operator/v1/vtcluster_types.go +++ b/api/operator/v1/vtcluster_types.go @@ -674,7 +674,11 @@ func (cr *VTCluster) Validate() error { } storageNodes := sets.New[string]() if cr.Spec.Storage != nil { - storageNodes.Insert(cr.AsURL(vmv1beta1.ClusterComponentStorage, false)) + storageURL, err := cr.AsURL(vmv1beta1.ClusterComponentStorage, false) + if err != nil { + return err + } + storageNodes.Insert(storageURL) vts := cr.Spec.Storage name := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) if vts.ServiceSpec != nil && vts.ServiceSpec.Name == name { @@ -796,14 +800,14 @@ func (cr *VTCluster) IsOwnsServiceAccount() bool { // AsURL implements stub for interface. // nolint:dupl,lll -func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string { +func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) (string, error) { var defaultPort string var svcSpec *vmv1beta1.AdditionalServiceSpec var extraArgs map[string]string switch kind { case vmv1beta1.ClusterComponentSelect: if cr.Spec.Select == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.select configured", cr.Name) } defaultPort = "10471" if cr.Spec.Select.Port != "" { @@ -813,7 +817,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.Select.ExtraArgs case vmv1beta1.ClusterComponentInsert: if cr.Spec.Insert == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.insert configured", cr.Name) } defaultPort = "10481" if cr.Spec.Insert.Port != "" { @@ -823,7 +827,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string extraArgs = cr.Spec.Insert.ExtraArgs case vmv1beta1.ClusterComponentStorage: if cr.Spec.Storage == nil { - return "" + return "", fmt.Errorf("vtcluster %q has no spec.storage configured", cr.Name) } defaultPort = "10491" if cr.Spec.Storage.Port != "" { @@ -835,7 +839,7 @@ func (cr *VTCluster) AsURL(kind vmv1beta1.ClusterComponent, isExtra bool) string panic("BUG unsupported cluster kind=" + string(kind)) } svcName, port := vmv1beta1.ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + return fmt.Sprintf("%s://%s.%s.svc:%s", vmv1beta1.HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port), nil } // +kubebuilder:object:root=true diff --git a/api/operator/v1alpha1/vmdistributed_types.go b/api/operator/v1alpha1/vmdistributed_types.go index 369f077afe..4c0c6df1b4 100644 --- a/api/operator/v1alpha1/vmdistributed_types.go +++ b/api/operator/v1alpha1/vmdistributed_types.go @@ -185,7 +185,7 @@ type VMDistributedZoneCluster struct { // +optional // +kubebuilder:validation:Schemaless // +kubebuilder:pruning:PreserveUnknownFields - Spec vmv1beta1.VMClusterSpec `json:"spec"` + Spec vmv1beta1.VMClusterSpecBase `json:"spec"` } // +k8s:openapi-gen=true @@ -517,7 +517,7 @@ func (cr *VMDistributed) Validate() error { hasCommonVMInsert := cr.Spec.ZoneCommon.VMCluster.Spec.VMInsert != nil hasCommonVMSelect := cr.Spec.ZoneCommon.VMCluster.Spec.VMSelect != nil hasCommonVMSingle := cr.Spec.ZoneCommon.VMSingle != nil && (cr.Spec.ZoneCommon.VMSingle.Name != "" || cr.Spec.ZoneCommon.VMSingle.Spec != nil) - hasCommonVMCluster := cr.Spec.ZoneCommon.VMCluster.Name != "" || !equality.Semantic.DeepEqual(cr.Spec.ZoneCommon.VMCluster.Spec, vmv1beta1.VMClusterSpec{}) + hasCommonVMCluster := cr.Spec.ZoneCommon.VMCluster.Name != "" || !equality.Semantic.DeepEqual(cr.Spec.ZoneCommon.VMCluster.Spec, vmv1beta1.VMClusterSpecBase{}) if isVMSingle && hasCommonVMCluster { return fmt.Errorf("backendType=VMSingle is incompatible with zoneCommon.vmcluster configuration") } @@ -534,7 +534,7 @@ func (cr *VMDistributed) Validate() error { } zones.Insert(zone.Name) if isVMSingle { - if zone.VMCluster.Name != "" || !equality.Semantic.DeepEqual(zone.VMCluster.Spec, vmv1beta1.VMClusterSpec{}) { + if zone.VMCluster.Name != "" || !equality.Semantic.DeepEqual(zone.VMCluster.Spec, vmv1beta1.VMClusterSpecBase{}) { return fmt.Errorf("spec.zones[%d]: backendType=VMSingle is incompatible with vmcluster configuration", i) } singleName := zone.VMSingleName(cr) diff --git a/api/operator/v1alpha1/vmdistributed_types_test.go b/api/operator/v1alpha1/vmdistributed_types_test.go index 728096dd33..944ee8e43b 100644 --- a/api/operator/v1alpha1/vmdistributed_types_test.go +++ b/api/operator/v1alpha1/vmdistributed_types_test.go @@ -121,7 +121,7 @@ func TestValidateVMDistributed(t *testing.T) { { Name: "zone-1", VMSingle: &VMDistributedZoneSingle{Name: "single-a"}, - VMCluster: VMDistributedZoneCluster{Spec: vmv1beta1.VMClusterSpec{ + VMCluster: VMDistributedZoneCluster{Spec: vmv1beta1.VMClusterSpecBase{ VMInsert: &vmv1beta1.VMInsert{}, }}, }, @@ -143,7 +143,7 @@ func TestValidateVMDistributed(t *testing.T) { VMSingle: &VMDistributedZoneSingle{ Spec: &vmv1beta1.VMSingleSpec{}, }, - VMCluster: VMDistributedZoneCluster{Spec: vmv1beta1.VMClusterSpec{ + VMCluster: VMDistributedZoneCluster{Spec: vmv1beta1.VMClusterSpecBase{ VMInsert: &vmv1beta1.VMInsert{}, }}, }, @@ -177,6 +177,27 @@ func TestValidateVMDistributed(t *testing.T) { }, isErr: true, }) + + // backendType=VMCluster with a valid shared vminsert/vmselect - no error + f(opts{ + cr: VMDistributed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: VMDistributedSpec{ + ZoneCommon: VMDistributedZoneCommon{ + VMCluster: VMDistributedZoneCluster{Spec: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{}, + VMSelect: &vmv1beta1.VMSelect{}, + }}, + }, + Zones: []VMDistributedZone{ + {Name: "zone-1"}, + }, + }, + }, + isErr: false, + }) } func TestEnsureNoVMOwners(t *testing.T) { diff --git a/api/operator/v1beta1/vmcluster_types.go b/api/operator/v1beta1/vmcluster_types.go index a9758d4388..a8744add4d 100644 --- a/api/operator/v1beta1/vmcluster_types.go +++ b/api/operator/v1beta1/vmcluster_types.go @@ -13,12 +13,31 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/utils/ptr" ) // VMClusterSpec defines the desired state of VMCluster // +k8s:openapi-gen=true type VMClusterSpec struct { + VMClusterSpecBase `json:",inline"` + + // Pools defines named groups of vmstorage (and optionally vminsert) components. + // Each pool gets its own StatefulSet and headless Service named --. + // Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field. + // vmselect queries all pools using the pool name as a storage group name (-storageNode=/). + // When pools are defined the top-level vmstorage is not deployed; pools replace it entirely. + // The top-level vminsert is deployed as a shared insert group across all pools only when no pool + // defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. + // +optional + // +listType=map + // +listMapKey=name + Pools []VMClusterPool `json:"pools,omitempty"` +} + +// VMClusterSpecBase defines the desired state of VMCluster, excluding Pools. +// +k8s:openapi-gen=true +type VMClusterSpecBase struct { // RetentionPeriod defines how long to retain stored metrics, specified as a duration (e.g., "1d", "1w", "1m"). // Data with timestamps outside the RetentionPeriod is automatically deleted. The minimum allowed value is 1d, or 24h. // The default value is 1 (one month). @@ -30,6 +49,14 @@ type VMClusterSpec struct { // distinct storage nodes // +optional ReplicationFactor *int32 `json:"replicationFactor,omitempty"` + // GlobalReplicationFactor sets vmselect's -globalReplicationFactor, i.e. how many copies of + // every ingested sample are available across distinct storage groups (see Pools). vmselect + // continues returning full responses when up to GlobalReplicationFactor-1 groups are + // temporarily unavailable. Only meaningful when Pools is set and samples are actually + // replicated across pools (e.g. via a remote-write fan-out upstream of the cluster). + // See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmstorage-groups-at-vmselect + // +optional + GlobalReplicationFactor *int32 `json:"globalReplicationFactor,omitempty"` // ServiceAccountName is the name of the ServiceAccount to use to run the // VMSelect, VMStorage and VMInsert Pods. @@ -134,7 +161,16 @@ func (d *VMClusterDiscovery) enabled() bool { return d != nil && d.Enabled } -func (d *VMClusterDiscovery) validate() error { +func (d *VMClusterDiscovery) validate(license *License) error { + if !d.enabled() { + return nil + } + if !license.IsProvided() { + return fmt.Errorf("discovery requires a valid license key, see https://docs.victoriametrics.com/victoriametrics/enterprise/") + } + if err := license.validate(); err != nil { + return err + } if len(d.Filter) > 0 { if _, err := regexp.Compile(d.Filter); err != nil { return fmt.Errorf("discovery.filter is not a valid regexp: %w", err) @@ -526,6 +562,11 @@ type VMStorage struct { // it can be overwritten with component specific image.tag value. // +optional ComponentVersion string `json:"componentVersion,omitempty"` + // RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance. + // Useful when using Pools to implement multi-retention setups. + // +optional + // +kubebuilder:validation:Pattern:="^[0-9]+(h|d|w|y)?$" + RetentionPeriod string `json:"retentionPeriod,omitempty"` // PodMetadata configures Labels and Annotations which are propagated to the VMStorage pods. PodMetadata *EmbeddedObjectMetadata `json:"podMetadata,omitempty"` // LogFormat for VMStorage to be configured with. @@ -747,10 +788,64 @@ func (cr *VMCluster) GetRemoteWriteURL() string { if cr == nil || cr.Spec.VMInsert == nil { return "" } - insertURL := cr.AsURL(ClusterComponentInsert, false) + insertURL, err := cr.AsURLForPool(ClusterComponentInsert, "", false) + if err != nil { + return "" + } return fmt.Sprintf("%s%s", insertURL, BuildPathWithPrefixFlag(cr.Spec.VMInsert.ExtraArgs, "/insert/multitenant/prometheus/api/v1/write")) } +func (vms *VMStorage) validate(license *License, clusterRetentionPeriod string) error { + if err := vms.ServiceSpec.ValidateNoServiceTypeOverrideWithUseAsDefault(); err != nil { + return err + } + if vms.VMBackup != nil { + if err := vms.VMBackup.validate(license); err != nil { + return err + } + } + retention := clusterRetentionPeriod + if vms.RetentionPeriod != "" { + retention = vms.RetentionPeriod + } + if err := vms.RetentionFilters.validate(license, retention); err != nil { + return err + } + if vms.HPA != nil { + if vms.HPA.Behaviour != nil && vms.HPA.Behaviour.ScaleDown != nil { + return fmt.Errorf("scaledown HPA behavior is not supported") + } + if err := vms.HPA.Validate(); err != nil { + return err + } + } + if vms.VPA != nil { + if err := vms.VPA.Validate(); err != nil { + return err + } + } + if vms.RollingUpdateStrategyBehavior != nil { + if err := vms.RollingUpdateStrategyBehavior.Validate(); err != nil { + return err + } + } + return vms.Validate() +} + +func (vmi *VMInsert) validate() error { + if vmi.HPA != nil { + if err := vmi.HPA.Validate(); err != nil { + return err + } + } + if vmi.VPA != nil { + if err := vmi.VPA.Validate(); err != nil { + return err + } + } + return vmi.Validate() +} + func (cr *VMCluster) Validate() error { if MustSkipCRValidation(cr) { return nil @@ -783,10 +878,25 @@ func (cr *VMCluster) Validate() error { return fmt.Errorf("vmselect: %w", err) } storageNodes := sets.New[string]() - if cr.Spec.VMStorage != nil && !vms.Discovery.OrDefault(cr.Spec.Discovery).enabled() { - storageName := cr.PrefixedName(ClusterComponentStorage) - for _, idx := range cr.AvailableStorageNodeIDs(ClusterComponentSelect) { - storageNodes.Insert(PodDNSAddress(storageName, idx, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)) + if !vms.Discovery.OrDefault(cr.Spec.Discovery).enabled() { + if cr.Spec.VMStorage != nil && len(cr.Spec.Pools) == 0 { + storageName := cr.PrefixedName(ClusterComponentStorage) + for _, idx := range cr.AvailableStorageNodeIDs(ClusterComponentSelect) { + storageNodes.Insert(PodDNSAddress(storageName, idx, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)) + } + } + for _, pool := range cr.Spec.Pools { + poolVMStorage, err := cr.ResolvePoolVMStorage(&pool) + if err != nil { + return fmt.Errorf("pools[%q] vmstorage merge: %w", pool.Name, err) + } + if poolVMStorage == nil { + continue + } + poolStorageName := cr.PoolPrefixedName(ClusterComponentStorage, pool.Name) + for _, idx := range AvailableStorageNodeIDsFor(poolVMStorage, ClusterComponentSelect) { + storageNodes.Insert(PodDNSAddress(poolStorageName, idx, cr.Namespace, poolVMStorage.VMSelectPort, cr.Spec.ClusterDomainName)) + } } } if nodes, ok := vms.ExtraArgs["storageNode"]; ok { @@ -814,17 +924,7 @@ func (cr *VMCluster) Validate() error { if vmi.ServiceSpec != nil && vmi.ServiceSpec.Name == name { return fmt.Errorf(".serviceSpec.Name cannot be equal to prefixed name=%q", name) } - if vmi.HPA != nil { - if err := vmi.HPA.Validate(); err != nil { - return err - } - } - if vmi.VPA != nil { - if err := vmi.VPA.Validate(); err != nil { - return err - } - } - if err := vmi.Validate(); err != nil { + if err := vmi.validate(); err != nil { return fmt.Errorf("vminsert: %w", err) } } @@ -837,31 +937,7 @@ func (cr *VMCluster) Validate() error { if vms.ServiceSpec != nil && vms.ServiceSpec.Name == name { return fmt.Errorf(".serviceSpec.Name cannot be equal to prefixed name=%q", name) } - if err := vms.ServiceSpec.ValidateNoServiceTypeOverrideWithUseAsDefault(); err != nil { - return err - } - if cr.Spec.VMStorage.VMBackup != nil { - if err := cr.Spec.VMStorage.VMBackup.validate(cr.Spec.License); err != nil { - return err - } - } - if err := vms.RetentionFilters.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { - return err - } - if vms.RollingUpdateStrategyBehavior != nil { - if err := vms.RollingUpdateStrategyBehavior.Validate(); err != nil { - return fmt.Errorf("vmstorage: %w", err) - } - } - if vms.HPA != nil && vms.HPA.Behaviour != nil && vms.HPA.Behaviour.ScaleDown != nil { - return fmt.Errorf("vmstorage scaledown HPA behavior is not supported") - } - if vms.VPA != nil { - if err := vms.VPA.Validate(); err != nil { - return err - } - } - if err := vms.Validate(); err != nil { + if err := vms.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { return fmt.Errorf("vmstorage: %w", err) } } @@ -882,19 +958,13 @@ func (cr *VMCluster) Validate() error { if cr.Spec.VMSelect != nil { vmselectDiscovery = cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery) } - if vminsertDiscovery.enabled() || vmselectDiscovery.enabled() { - if !cr.Spec.License.IsProvided() { - return fmt.Errorf("discovery requires a valid license key, see https://docs.victoriametrics.com/victoriametrics/enterprise/") - } - if err := cr.Spec.License.validate(); err != nil { - return err - } - } - if vminsertDiscovery.enabled() { - if cr.Spec.VMStorage != nil && len(cr.Spec.VMStorage.MaintenanceInsertNodeIDs) > 0 { - return fmt.Errorf("maintenanceInsertNodeIDs cannot be used when vminsert discovery is enabled") + if !HasAnyPoolInsert(cr.Spec.Pools) { + if vminsertDiscovery.enabled() { + if cr.Spec.VMStorage != nil && len(cr.Spec.VMStorage.MaintenanceInsertNodeIDs) > 0 { + return fmt.Errorf("maintenanceInsertNodeIDs cannot be used when vminsert discovery is enabled") + } } - if err := vminsertDiscovery.validate(); err != nil { + if err := vminsertDiscovery.validate(cr.Spec.License); err != nil { return fmt.Errorf("vminsert: %w", err) } } @@ -902,8 +972,79 @@ func (cr *VMCluster) Validate() error { if cr.Spec.VMStorage != nil && len(cr.Spec.VMStorage.MaintenanceSelectNodeIDs) > 0 { return fmt.Errorf("maintenanceSelectNodeIDs cannot be used when vmselect discovery is enabled") } - if err := vmselectDiscovery.validate(); err != nil { - return fmt.Errorf("vmselect: %w", err) + } + if err := vmselectDiscovery.validate(cr.Spec.License); err != nil { + return fmt.Errorf("vmselect: %w", err) + } + + poolNames := make(map[string]struct{}, len(cr.Spec.Pools)) + var hasPoolInsert bool + for i, pool := range cr.Spec.Pools { + if errs := validation.IsDNS1123Label(pool.Name); len(errs) > 0 { + return fmt.Errorf("pools[%d].name %q is invalid: %s", i, pool.Name, strings.Join(errs, "; ")) + } + if len(pool.Name) > maxVMClusterPoolNameLength { + return fmt.Errorf("pools[%d].name %q is too long: max %d characters", i, pool.Name, maxVMClusterPoolNameLength) + } + for _, kind := range []ClusterComponent{ClusterComponentStorage, ClusterComponentInsert} { + if name := cr.PoolPrefixedName(kind, pool.Name); len(name) > 63 { + return fmt.Errorf("pools[%d].name %q: generated name %q exceeds Kubernetes 63-character limit, use a shorter cluster or pool name", i, pool.Name, name) + } + } + if _, dup := poolNames[pool.Name]; dup { + return fmt.Errorf("pools[%d].name %q is duplicated", i, pool.Name) + } + poolNames[pool.Name] = struct{}{} + poolVMStorage, err := cr.ResolvePoolVMStorage(&pool) + if err != nil { + return fmt.Errorf("pools[%d] vmstorage merge: %w", i, err) + } + if poolVMStorage == nil { + return fmt.Errorf("pools[%d] %q: vmstorage is not defined at either the pool or cluster level", i, pool.Name) + } + if poolStorageName := cr.PoolPrefixedName(ClusterComponentStorage, pool.Name); poolVMStorage.ServiceSpec != nil && poolVMStorage.ServiceSpec.Name == poolStorageName { + return fmt.Errorf("pools[%d] %q: vmstorage.serviceSpec.Name cannot be equal to prefixed name=%q", i, pool.Name, poolStorageName) + } + if pool.VMStorage != nil { + if err := poolVMStorage.validate(cr.Spec.License, cr.Spec.RetentionPeriod); err != nil { + return fmt.Errorf("pools[%d] vmstorage: %w", i, err) + } + } + if vmselectDiscovery.enabled() && len(poolVMStorage.MaintenanceSelectNodeIDs) > 0 { + return fmt.Errorf("pools[%d] %q: maintenanceSelectNodeIDs cannot be used when vmselect discovery is enabled", i, pool.Name) + } + if pool.VMInsert != nil { + hasPoolInsert = true + vmi, err := cr.ResolvePoolVMInsert(&pool) + if err != nil { + return fmt.Errorf("pools[%d] vminsert merge: %w", i, err) + } + if err := vmi.validate(); err != nil { + return fmt.Errorf("pools[%d] vminsert: %w", i, err) + } + if poolInsertName := cr.PoolPrefixedName(ClusterComponentInsert, pool.Name); vmi.ServiceSpec != nil && vmi.ServiceSpec.Name == poolInsertName { + return fmt.Errorf("pools[%d] %q: vminsert.serviceSpec.Name cannot be equal to prefixed name=%q", i, pool.Name, poolInsertName) + } + poolDiscovery := vmi.Discovery.OrDefault(cr.Spec.Discovery) + if poolDiscovery.enabled() && len(poolVMStorage.MaintenanceInsertNodeIDs) > 0 { + return fmt.Errorf("pools[%d] %q: maintenanceInsertNodeIDs cannot be used when vminsert discovery is enabled", i, pool.Name) + } + if err := poolDiscovery.validate(cr.Spec.License); err != nil { + return fmt.Errorf("pools[%d] vminsert: %w", i, err) + } + } else if vminsertDiscovery.enabled() && len(poolVMStorage.MaintenanceInsertNodeIDs) > 0 { + return fmt.Errorf("pools[%d] %q: maintenanceInsertNodeIDs cannot be used when vminsert discovery is enabled", i, pool.Name) + } + } + if hasPoolInsert { + for i, pool := range cr.Spec.Pools { + if pool.VMInsert == nil { + return fmt.Errorf("pools[%d] %q: vminsert must be defined once any pool defines its own vminsert, otherwise this pool has no ingestion path", i, pool.Name) + } + } + if cr.Spec.RequestsLoadBalancer.Enabled && !cr.Spec.RequestsLoadBalancer.DisableInsertBalancing && len(cr.Spec.Pools) > 1 { + return fmt.Errorf("requestsLoadBalancer for vminsert is not supported with more than one pool defining its own vminsert: " + + "all such pools would reconcile the same load-balanced service name, causing them to overwrite each other") } } @@ -912,24 +1053,30 @@ func (cr *VMCluster) Validate() error { // AvailableStorageNodeIDs returns ids of the storage nodes for the provided component func (cr *VMCluster) AvailableStorageNodeIDs(kind ClusterComponent) []int32 { + return AvailableStorageNodeIDsFor(cr.Spec.VMStorage, kind) +} + +// AvailableStorageNodeIDsFor returns ids of the storage nodes for an arbitrary VMStorage +// (either the cluster's top-level VMStorage or a pool's resolved VMStorage). +func AvailableStorageNodeIDsFor(storage *VMStorage, kind ClusterComponent) []int32 { var result []int32 - if cr.Spec.VMStorage == nil || (cr.Spec.VMStorage.ReplicaCount == nil && cr.Spec.VMStorage.HPA == nil) { + if storage == nil || (storage.ReplicaCount == nil && storage.HPA == nil) { return result } maintenanceNodes := sets.New[int32]() switch kind { case ClusterComponentSelect: - maintenanceNodes.Insert(cr.Spec.VMStorage.MaintenanceSelectNodeIDs...) + maintenanceNodes.Insert(storage.MaintenanceSelectNodeIDs...) case ClusterComponentInsert: - maintenanceNodes.Insert(cr.Spec.VMStorage.MaintenanceInsertNodeIDs...) + maintenanceNodes.Insert(storage.MaintenanceInsertNodeIDs...) default: panic("BUG unsupported kind: " + string(kind)) } var replicaCount int32 - if cr.Spec.VMStorage.ReplicaCount != nil { - replicaCount = *cr.Spec.VMStorage.ReplicaCount - } else if cr.Spec.VMStorage.HPA != nil { - replicaCount = cr.Spec.VMStorage.HPA.GetMinReplicas() + if storage.ReplicaCount != nil { + replicaCount = *storage.ReplicaCount + } else if storage.HPA != nil { + replicaCount = storage.HPA.GetMinReplicas() } for i := int32(0); i < replicaCount; i++ { if maintenanceNodes.Has(i) { @@ -1066,15 +1213,81 @@ func (cr *VMCluster) IsOwnsServiceAccount() bool { return cr.Spec.ServiceAccountName == "" } -// AsURL implements stub for interface. +// FindPool returns the pool with the given name, if any. +func (cr *VMCluster) FindPool(name string) (*VMClusterPool, bool) { + for i := range cr.Spec.Pools { + if cr.Spec.Pools[i].Name == name { + return &cr.Spec.Pools[i], true + } + } + return nil, false +} + +// HasAnyPoolInsert reports whether any pool defines its own dedicated vminsert, which is +// exactly the condition under which the operator skips creating the shared top-level vminsert. +func HasAnyPoolInsert(pools []VMClusterPool) bool { + for _, p := range pools { + if p.VMInsert != nil { + return true + } + } + return false +} + +// mergedPoolInsert merges pool.VMInsert over the top-level base, mirroring how the operator +// resolves a pool's own vminsert when reconciling it. +func (cr *VMCluster) mergedPoolInsert(pool *VMClusterPool) (*VMInsert, error) { + merged := pool.VMInsert.DeepCopy() + if cr.Spec.VMInsert != nil { + if err := MergeDeep(merged, cr.Spec.VMInsert, true); err != nil { + return nil, fmt.Errorf("pool %q vminsert merge: %w", pool.Name, err) + } + } + return merged, nil +} + +// mergedPoolStorage merges pool.VMStorage over the top-level base, mirroring how the operator +// resolves a pool's own vmstorage when reconciling it. A pool without its own override falls +// back to the base as-is. +func (cr *VMCluster) mergedPoolStorage(pool *VMClusterPool) (*VMStorage, error) { + if pool.VMStorage == nil { + return cr.Spec.VMStorage, nil + } + merged := pool.VMStorage.DeepCopy() + if cr.Spec.VMStorage != nil { + if err := MergeDeep(merged, cr.Spec.VMStorage, true); err != nil { + return nil, fmt.Errorf("pool %q vmstorage merge: %w", pool.Name, err) + } + } + return merged, nil +} + +// AsURL builds the Service URL for the given cluster component's shared (non-pool) instance. +// Use AsURLForPool to resolve a specific pool's Service URL instead. func (cr *VMCluster) AsURL(kind ClusterComponent, isExtra bool) string { + url, _ := cr.AsURLForPool(kind, "", isExtra) + return url +} + +// AsURLForPool builds the Service URL for the given cluster component. poolName selects a +// specific entry from spec.pools instead of the shared (non-pool) component; it's only +// meaningful for ClusterComponentInsert and ClusterComponentStorage, since vmselect is never +// pool-scoped. +// It errors instead of silently returning a URL for a Service the operator wouldn't actually +// create - e.g. the shared vminsert once any pool defines its own, or the top-level vmstorage +// once any pool is defined at all. +func (cr *VMCluster) AsURLForPool(kind ClusterComponent, poolName string, isExtra bool) (string, error) { var defaultPort string var svcSpec *AdditionalServiceSpec var extraArgs map[string]string + svcName := cr.PrefixedName(kind) switch kind { case ClusterComponentSelect: + if poolName != "" { + return "", fmt.Errorf("vmcluster %q: pool %q is not applicable to vmselect, since vmselect is never pool-scoped", cr.Name, poolName) + } if cr.Spec.VMSelect == nil { - return "" + return "", fmt.Errorf("vmcluster %q has no spec.vmSelect configured", cr.Name) } defaultPort = "8481" if cr.Spec.VMSelect.Port != "" { @@ -1083,30 +1296,64 @@ func (cr *VMCluster) AsURL(kind ClusterComponent, isExtra bool) string { svcSpec = cr.Spec.VMSelect.ServiceSpec extraArgs = cr.Spec.VMSelect.ExtraArgs case ClusterComponentInsert: - if cr.Spec.VMInsert == nil { - return "" - } defaultPort = "8480" - if cr.Spec.VMInsert.Port != "" { - defaultPort = cr.Spec.VMInsert.Port + vmi := cr.Spec.VMInsert + if poolName != "" { + pool, ok := cr.FindPool(poolName) + if !ok { + return "", fmt.Errorf("vmcluster %q has no pool named %q", cr.Name, poolName) + } + if pool.VMInsert != nil { + merged, err := cr.mergedPoolInsert(pool) + if err != nil { + return "", err + } + vmi = merged + svcName = cr.PoolPrefixedName(kind, poolName) + } else if HasAnyPoolInsert(cr.Spec.Pools) { + return "", fmt.Errorf("vmcluster %q: pool %q has no dedicated vmInsert, and the shared vmInsert is not deployed because another pool defines its own", cr.Name, poolName) + } + } else if HasAnyPoolInsert(cr.Spec.Pools) { + return "", fmt.Errorf("vmcluster %q has per-pool vminsert(s); target a specific pool instead of the shared vminsert", cr.Name) } - svcSpec = cr.Spec.VMInsert.ServiceSpec - extraArgs = cr.Spec.VMInsert.ExtraArgs - case ClusterComponentStorage: - if cr.Spec.VMStorage == nil { - return "" + if vmi == nil { + return "", fmt.Errorf("vmcluster %q has no shared spec.vmInsert configured", cr.Name) + } + if vmi.Port != "" { + defaultPort = vmi.Port } + svcSpec = vmi.ServiceSpec + extraArgs = vmi.ExtraArgs + case ClusterComponentStorage: defaultPort = "8482" - if cr.Spec.VMStorage.Port != "" { - defaultPort = cr.Spec.VMStorage.Port + vms := cr.Spec.VMStorage + if poolName != "" { + pool, ok := cr.FindPool(poolName) + if !ok { + return "", fmt.Errorf("vmcluster %q has no pool named %q", cr.Name, poolName) + } + merged, err := cr.mergedPoolStorage(pool) + if err != nil { + return "", err + } + vms = merged + svcName = cr.PoolPrefixedName(kind, poolName) + } else if len(cr.Spec.Pools) > 0 { + return "", fmt.Errorf("vmcluster %q defines pools; target a specific pool instead of the top-level vmstorage", cr.Name) + } + if vms == nil { + return "", fmt.Errorf("vmcluster %q has no shared spec.vmStorage configured", cr.Name) } - svcSpec = cr.Spec.VMStorage.ServiceSpec - extraArgs = cr.Spec.VMStorage.ExtraArgs + if vms.Port != "" { + defaultPort = vms.Port + } + svcSpec = vms.ServiceSpec + extraArgs = vms.ExtraArgs default: panic("BUG unsupported cluster kind=" + string(kind)) } - svcName, port := ResolveServiceURL(cr.PrefixedName(kind), defaultPort, "http", svcSpec, isExtra) - return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(extraArgs), svcName, cr.Namespace, port) + resolvedName, port := ResolveServiceURL(svcName, defaultPort, "http", svcSpec, isExtra) + return fmt.Sprintf("%s://%s.%s.svc:%s", HTTPProtoFromFlags(extraArgs), resolvedName, cr.Namespace, port), nil } func (cr *VMSelect) ProbePath() string { @@ -1249,3 +1496,78 @@ func (cr *VMAuthLoadBalancerSpec) UseTLS() bool { func (cr *VMAuthLoadBalancerSpec) GetMetricsPath() string { return BuildPathWithPrefixFlag(cr.ExtraArgs, metricsPath) } + +// maxVMClusterPoolNameLength must match the kubebuilder MaxLength on VMClusterPool.Name below. +const maxVMClusterPoolNameLength = 16 + +// VMClusterPool defines a named group of vmstorage (and optionally vminsert) components +// within a VMCluster. Each pool gets its own StatefulSet and headless Service. +// +k8s:openapi-gen=true +type VMClusterPool struct { + // Name is the unique identifier for this pool within the cluster. + // Used as a suffix for generated resource names (e.g. vmstorage--) and as a + // storage group name in vmselect. Kept short since the cluster name itself isn't length-limited, + // and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit. + // Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. + // +kubebuilder:validation:Pattern:="^[a-z0-9]([a-z0-9-]*[a-z0-9])?$" + // +kubebuilder:validation:MaxLength=16 + Name string `json:"name"` + // VMStorage defines pool-specific vmstorage configuration. + // Each field overrides the corresponding field in the top-level vmstorage spec. + // Fields absent here inherit from the top-level vmstorage. + // RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. + // +optional + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + VMStorage *VMStorage `json:"vmstorage,omitempty"` + // VMInsert defines a dedicated vminsert for this pool. + // Each field overrides the corresponding field in the top-level vminsert spec. + // When nil, the top-level shared vminsert writes to this pool's storage nodes as well. + // +optional + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + VMInsert *VMInsert `json:"vminsert,omitempty"` + // ReplicationFactor overrides the cluster-level replicationFactor for this pool's storage + // group at vmselect, rendered as -replicationFactor=:. Pools + // without an override use the cluster-level ReplicationFactor, if set. + // +optional + ReplicationFactor *int32 `json:"replicationFactor,omitempty"` +} + +// PoolPrefixedName returns the Kubernetes resource name for the given component in a pool. +func (cr *VMCluster) PoolPrefixedName(kind ClusterComponent, poolName string) string { + return cr.PrefixedName(kind) + "-" + poolName +} + +// ResolvePoolVMStorage merges pool's vmstorage over the cluster's top-level vmstorage. +func (cr *VMCluster) ResolvePoolVMStorage(pool *VMClusterPool) (*VMStorage, error) { + base := cr.Spec.VMStorage + if pool == nil || pool.VMStorage == nil { + if base == nil { + return nil, nil + } + return base.DeepCopy(), nil + } + merged := pool.VMStorage.DeepCopy() + if base != nil { + if err := MergeDeep(merged, base, true); err != nil { + return nil, err + } + } + return merged, nil +} + +// ResolvePoolVMInsert merges pool's vminsert over the cluster's top-level vminsert. +func (cr *VMCluster) ResolvePoolVMInsert(pool *VMClusterPool) (*VMInsert, error) { + if pool == nil || pool.VMInsert == nil { + return nil, nil + } + base := cr.Spec.VMInsert + merged := pool.VMInsert.DeepCopy() + if base != nil { + if err := MergeDeep(merged, base, true); err != nil { + return nil, err + } + } + return merged, nil +} diff --git a/api/operator/v1beta1/vmcluster_types_test.go b/api/operator/v1beta1/vmcluster_types_test.go index 14dfde062a..320b9683e0 100644 --- a/api/operator/v1beta1/vmcluster_types_test.go +++ b/api/operator/v1beta1/vmcluster_types_test.go @@ -98,12 +98,14 @@ func TestVMCluster_AvailableStorageNodeIDs(t *testing.T) { cr := &VMCluster{ Spec: VMClusterSpec{ - VMStorage: &VMStorage{ - CommonAppsParams: CommonAppsParams{ - ReplicaCount: ptr.To(int32(5)), + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(5)), + }, + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 4}, }, - MaintenanceSelectNodeIDs: []int32{1, 3}, - MaintenanceInsertNodeIDs: []int32{0, 4}, }, }, } @@ -117,8 +119,10 @@ func TestVMCluster_AvailableStorageNodeIDs(t *testing.T) { // no maintenance nodes f(&VMCluster{ Spec: VMClusterSpec{ - VMStorage: &VMStorage{ - CommonAppsParams: CommonAppsParams{ReplicaCount: ptr.To(int32(3))}, + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + CommonAppsParams: CommonAppsParams{ReplicaCount: ptr.To(int32(3))}, + }, }, }, }, ClusterComponentSelect, []int32{0, 1, 2}) @@ -140,263 +144,325 @@ func TestVMCluster_Validate(t *testing.T) { // downsampling without license f(VMClusterSpec{ - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + VMClusterSpecBase: VMClusterSpecBase{ + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + }, }, }, true) // downsampling with valid config f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + }, }, }, false) // downsampling with filter and dedupInterval f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Filter: `{env="prod"}`, Periods: []DownsamplingPeriod{{Offset: "90d", Interval: "1h"}}}}, - DedupInterval: "1m", + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Filter: `{env="prod"}`, Periods: []DownsamplingPeriod{{Offset: "90d", Interval: "1h"}}}}, + DedupInterval: "1m", + }, }, }, false) // downsampling - multiple periods per rule f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{ - {Offset: "30d", Interval: "10m"}, - {Offset: "180d", Interval: "1h"}, - }}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{ + {Offset: "30d", Interval: "10m"}, + {Offset: "180d", Interval: "1h"}, + }}}, + }, }, }, false) // downsampling - duplicate filter f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{ - {Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}, - {Periods: []DownsamplingPeriod{{Offset: "180d", Interval: "1h"}}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{ + {Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}, + {Periods: []DownsamplingPeriod{{Offset: "180d", Interval: "1h"}}}, + }, }, }, }, true) // downsampling - offset not a multiple of interval f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "1d", Interval: "7m"}}}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "1d", Interval: "7m"}}}}, + }, }, }, true) // downsampling - period interval not a multiple of dedupInterval f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, - DedupInterval: "7m", + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + DedupInterval: "7m", + }, }, }, true) // downsampling - period interval is a multiple of dedupInterval f(VMClusterSpec{ - License: testLicense, - Downsampling: &DownsamplingConfig{ - Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, - DedupInterval: "5m", + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Downsampling: &DownsamplingConfig{ + Rules: []DownsamplingRule{{Periods: []DownsamplingPeriod{{Offset: "30d", Interval: "10m"}}}}, + DedupInterval: "5m", + }, }, }, false) // retention filters without vmstorage section — no error (vmstorage is nil) f(VMClusterSpec{ - License: testLicense, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + }, }, false) // retention filters without license f(VMClusterSpec{ - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "3d"}}, + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "3d"}}, + }, }, }, true) // retention filters with valid config f(VMClusterSpec{ - License: testLicense, - RetentionPeriod: "30", - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "3d"}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + RetentionPeriod: "30", + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "3d"}}, + }, }, }, false) // retention filters - invalid filter f(VMClusterSpec{ - License: testLicense, - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: "not-a-filter", Retention: "3d"}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: "not-a-filter", Retention: "3d"}}, + }, }, }, true) // retention filters - invalid retention f(VMClusterSpec{ - License: testLicense, - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "bad"}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "bad"}}, + }, }, }, true) // retention filters - retention exceeds retentionPeriod f(VMClusterSpec{ - License: testLicense, - RetentionPeriod: "30d", - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "1y"}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + RetentionPeriod: "30d", + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "1y"}}, + }, }, }, true) // retention filters - retention equal to retentionPeriod is ok f(VMClusterSpec{ - License: testLicense, - RetentionPeriod: "1y", - VMStorage: &VMStorage{ - RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "1y"}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + RetentionPeriod: "1y", + VMStorage: &VMStorage{ + RetentionFilters: &RetentionFiltersConfig{{Filter: `{env="dev"}`, Retention: "1y"}}, + }, }, }, false) // discovery without license f(VMClusterSpec{ - VMInsert: &VMInsert{}, - Discovery: &VMClusterDiscovery{Enabled: true}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{}, + Discovery: &VMClusterDiscovery{Enabled: true}, + }, }, true) // discovery with license f(VMClusterSpec{ - VMInsert: &VMInsert{}, - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{}, + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + }, }, false) // discovery with invalid filter regexp f(VMClusterSpec{ - VMInsert: &VMInsert{}, - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true, Filter: "[invalid"}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{}, + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true, Filter: "[invalid"}, + }, }, true) // discovery with valid filter regexp f(VMClusterSpec{ - VMInsert: &VMInsert{}, - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true, Filter: `vmstorage-test-[0-3]\.`}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{}, + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true, Filter: `vmstorage-test-[0-3]\.`}, + }, }, false) // global discovery + maintenanceInsertNodeIDs f(VMClusterSpec{ - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true}, - VMInsert: &VMInsert{}, - VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{}, + VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + }, }, true) // global discovery + maintenanceSelectNodeIDs f(VMClusterSpec{ - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true}, - VMSelect: &VMSelect{}, - VMStorage: &VMStorage{MaintenanceSelectNodeIDs: []int32{1}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMSelect: &VMSelect{}, + VMStorage: &VMStorage{MaintenanceSelectNodeIDs: []int32{1}}, + }, }, true) // component override disables vmselect discovery: maintenanceSelectNodeIDs is allowed f(VMClusterSpec{ - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true}, - VMInsert: &VMInsert{}, - VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: false}}, - VMStorage: &VMStorage{MaintenanceSelectNodeIDs: []int32{1}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{}, + VMSelect: &VMSelect{Discovery: &VMClusterDiscovery{Enabled: false}}, + VMStorage: &VMStorage{MaintenanceSelectNodeIDs: []int32{1}}, + }, }, false) // component override disables vminsert discovery: maintenanceInsertNodeIDs is allowed f(VMClusterSpec{ - License: testLicense, - Discovery: &VMClusterDiscovery{Enabled: true}, - VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: false}}, - VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: false}}, + VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + }, }, false) // component-level discovery enabled without global, requires license f(VMClusterSpec{ - VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: true}}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: true}}, + }, }, true) // component-level discovery enabled with license f(VMClusterSpec{ - License: testLicense, - VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: true}}, + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + VMInsert: &VMInsert{Discovery: &VMClusterDiscovery{Enabled: true}}, + }, }, false) // extraStorageNodes with unique addresses f(VMClusterSpec{ - VMSelect: &VMSelect{ - ExtraStorageNodes: []VMStorageNode{ - {Addr: "localhost:10101"}, - {Addr: "localhost:10102"}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ExtraStorageNodes: []VMStorageNode{ + {Addr: "localhost:10101"}, + {Addr: "localhost:10102"}, + }, }, }, }, false) // extraStorageNodes with duplicate addresses f(VMClusterSpec{ - VMSelect: &VMSelect{ - ExtraStorageNodes: []VMStorageNode{ - {Addr: "localhost:10101"}, - {Addr: "localhost:10101"}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ExtraStorageNodes: []VMStorageNode{ + {Addr: "localhost:10101"}, + {Addr: "localhost:10101"}, + }, }, }, }, true) // extraStorageNodes duplicating extraArgs storageNode f(VMClusterSpec{ - VMSelect: &VMSelect{ - CommonAppsParams: CommonAppsParams{ - ExtraArgs: map[string]string{"storageNode": "localhost:10101"}, - }, - ExtraStorageNodes: []VMStorageNode{ - {Addr: "localhost:10101"}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ExtraArgs: map[string]string{"storageNode": "localhost:10101"}, + }, + ExtraStorageNodes: []VMStorageNode{ + {Addr: "localhost:10101"}, + }, }, }, }, true) // extraStorageNodes with an empty addr f(VMClusterSpec{ - VMSelect: &VMSelect{ - ExtraStorageNodes: []VMStorageNode{ - {Addr: ""}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ExtraStorageNodes: []VMStorageNode{ + {Addr: ""}, + }, }, }, }, true) // vmselect: useAsDefault with an explicit service type must be rejected (default service is headless) f(VMClusterSpec{ - VMSelect: &VMSelect{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + }, }, }, }, }, true) f(VMClusterSpec{ - VMSelect: &VMSelect{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, + }, }, }, }, @@ -404,11 +470,13 @@ func TestVMCluster_Validate(t *testing.T) { // vmselect: useAsDefault without an explicit type is allowed f(VMClusterSpec{ - VMSelect: &VMSelect{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{{Name: "http", Port: 8481}}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "http", Port: 8481}}, + }, }, }, }, @@ -416,21 +484,25 @@ func TestVMCluster_Validate(t *testing.T) { // vmselect: explicit type without useAsDefault creates a separate service - allowed f(VMClusterSpec{ - VMSelect: &VMSelect{ - ServiceSpec: &AdditionalServiceSpec{ - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{ + ServiceSpec: &AdditionalServiceSpec{ + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, }, }, }, false) // vmstorage: useAsDefault with cluster IP set must be rejected (default service is headless) f(VMClusterSpec{ - VMStorage: &VMStorage{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: "1.1.1.1", + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "1.1.1.1", + }, }, }, }, @@ -438,12 +510,14 @@ func TestVMCluster_Validate(t *testing.T) { // vmstorage: useAsDefault without cluster IP set must be allowed f(VMClusterSpec{ - VMStorage: &VMStorage{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: corev1.ClusterIPNone, + }, }, }, }, @@ -451,11 +525,13 @@ func TestVMCluster_Validate(t *testing.T) { // vmstorage: useAsDefault without an explicit type is allowed f(VMClusterSpec{ - VMStorage: &VMStorage{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{{Name: "vminsert", Port: 8480}}, + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Name: "vminsert", Port: 8480}}, + }, }, }, }, @@ -463,10 +539,12 @@ func TestVMCluster_Validate(t *testing.T) { // vminsert: useAsDefault with an explicit type is allowed (default service is not headless) f(VMClusterSpec{ - VMInsert: &VMInsert{ - ServiceSpec: &AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + VMClusterSpecBase: VMClusterSpecBase{ + VMInsert: &VMInsert{ + ServiceSpec: &AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + }, }, }, }, false) @@ -476,23 +554,148 @@ func TestVMCluster_Validate(t *testing.T) { cr := &VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: VMClusterSpec{ - VMStorage: &VMStorage{ - CommonAppsParams: CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - VMSelectPort: "8481", + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{ + CommonAppsParams: CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + VMSelectPort: "8481", + }, + VMSelect: &VMSelect{}, }, - VMSelect: &VMSelect{}, }, } managedAddr := PodDNSAddress(cr.PrefixedName(ClusterComponentStorage), 0, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName) cr.Spec.VMSelect.ExtraStorageNodes = []VMStorageNode{{Addr: managedAddr}} assert.Error(t, cr.Validate()) } + + // pools: no pool defines vminsert — shared top-level vminsert covers all of them, valid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + }, + Pools: []VMClusterPool{ + {Name: "hot"}, + {Name: "cold"}, + }, + }, false) + + // pools: every pool defines its own vminsert — valid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + }, + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + }, + }, false) + + // pools: only some pools define vminsert — the rest would have no ingestion path + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + }, + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold"}, + }, + }, true) + + // pools: a pool with neither its own vmstorage nor a cluster-level one is invalid + f(VMClusterSpec{ + Pools: []VMClusterPool{ + {Name: "hot"}, + }, + }, true) + + // pools: global vminsert discovery + a pool's merged maintenanceInsertNodeIDs is invalid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + }, + Pools: []VMClusterPool{ + { + Name: "hot", + VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + VMInsert: &VMInsert{}, + }, + }, + }, true) + + // pools: a pool using the shared vminsert with global vminsert discovery enabled and its + // own merged maintenanceInsertNodeIDs is invalid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMInsert: &VMInsert{}, + }, + Pools: []VMClusterPool{ + { + Name: "hot", + VMStorage: &VMStorage{MaintenanceInsertNodeIDs: []int32{0}}, + }, + }, + }, true) + + // pools: global vmselect discovery + a pool's merged maintenanceSelectNodeIDs is invalid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + License: testLicense, + Discovery: &VMClusterDiscovery{Enabled: true}, + VMSelect: &VMSelect{}, + }, + Pools: []VMClusterPool{ + { + Name: "hot", + VMStorage: &VMStorage{MaintenanceSelectNodeIDs: []int32{1}}, + }, + }, + }, true) + + // pools: requestsLoadBalancer for vminsert with more than one dedicated-insert pool is invalid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + RequestsLoadBalancer: VMAuthLoadBalancer{Enabled: true}, + }, + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + }, + }, true) + + // pools: requestsLoadBalancer for vminsert with a single dedicated-insert pool is valid + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + RequestsLoadBalancer: VMAuthLoadBalancer{Enabled: true}, + }, + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + }, + }, false) + + // pools: requestsLoadBalancer with insert balancing disabled tolerates multiple dedicated-insert pools + f(VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMStorage: &VMStorage{}, + RequestsLoadBalancer: VMAuthLoadBalancer{Enabled: true, DisableInsertBalancing: true}, + }, + Pools: []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + }, + }, false) } func TestVMCluster_PrefixedName(t *testing.T) { f := func(name string, omit bool, kind ClusterComponent, want string) { t.Helper() - cr := &VMCluster{Spec: VMClusterSpec{UseLegacyNaming: omit}} + cr := &VMCluster{Spec: VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{UseLegacyNaming: omit}, + }} cr.Name = name assert.Equal(t, want, cr.PrefixedName(kind)) } @@ -507,3 +710,80 @@ func TestVMCluster_PrefixedName(t *testing.T) { f("myapp", true, ClusterComponentInsert, "myapp-vminsert") f("myapp", true, ClusterComponentStorage, "myapp-vmstorage") } + +func TestVMCluster_AsURL(t *testing.T) { + newCR := func() *VMCluster { + return &VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster-1", Namespace: "default"}, + Spec: VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + VMSelect: &VMSelect{}, + VMInsert: &VMInsert{}, + VMStorage: &VMStorage{}, + }, + }, + } + } + fOK := func(cr *VMCluster, kind ClusterComponent, poolName string, wantHostPrefix string) { + t.Helper() + url, err := cr.AsURLForPool(kind, poolName, false) + assert.NoError(t, err) + assert.Contains(t, url, wantHostPrefix) + } + fErr := func(cr *VMCluster, kind ClusterComponent, poolName string) { + t.Helper() + _, err := cr.AsURLForPool(kind, poolName, false) + assert.Error(t, err) + } + + // no pools: shared components resolve normally. + cr := newCR() + fOK(cr, ClusterComponentSelect, "", "vmselect-cluster-1.") + fOK(cr, ClusterComponentInsert, "", "vminsert-cluster-1.") + fOK(cr, ClusterComponentStorage, "", "vmstorage-cluster-1.") + + // no pools: unset component errors instead of returning an empty/dangling URL. + cr = newCR() + cr.Spec.VMInsert = nil + fErr(cr, ClusterComponentInsert, "") + + // select is never pool-scoped. + cr = newCR() + fErr(cr, ClusterComponentSelect, "hot") + + // pools defined, none with a dedicated insert: shared insert still resolves for poolName="". + cr = newCR() + cr.Spec.Pools = []VMClusterPool{{Name: "hot"}, {Name: "cold"}} + fOK(cr, ClusterComponentInsert, "", "vminsert-cluster-1.") + // ...and also resolves when a pool without its own insert is named explicitly. + fOK(cr, ClusterComponentInsert, "hot", "vminsert-cluster-1.") + // top-level storage no longer exists once pools are defined. + fErr(cr, ClusterComponentStorage, "") + // but a named pool's own storage does. + fOK(cr, ClusterComponentStorage, "hot", "vmstorage-cluster-1-hot.") + // referencing an unknown pool is an error. + fErr(cr, ClusterComponentInsert, "unknown") + fErr(cr, ClusterComponentStorage, "unknown") + + // pools defined, every pool has its own dedicated insert: shared insert is ambiguous. + cr = newCR() + cr.Spec.Pools = []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold", VMInsert: &VMInsert{}}, + } + fErr(cr, ClusterComponentInsert, "") + fOK(cr, ClusterComponentInsert, "hot", "vminsert-cluster-1-hot.") + fOK(cr, ClusterComponentInsert, "cold", "vminsert-cluster-1-cold.") + + // mixed: one pool has its own dedicated insert, another doesn't. The shared + // top-level insert is not deployed in this case (any pool insert disables it), + // so the pool without its own insert has nothing to route to. + cr = newCR() + cr.Spec.Pools = []VMClusterPool{ + {Name: "hot", VMInsert: &VMInsert{}}, + {Name: "cold"}, + } + fOK(cr, ClusterComponentInsert, "hot", "vminsert-cluster-1-hot.") + fErr(cr, ClusterComponentInsert, "cold") + fErr(cr, ClusterComponentInsert, "") +} diff --git a/api/operator/v1beta1/vmextra_types.go b/api/operator/v1beta1/vmextra_types.go index d9d4a2bcb3..ef20bd88dd 100644 --- a/api/operator/v1beta1/vmextra_types.go +++ b/api/operator/v1beta1/vmextra_types.go @@ -1880,3 +1880,76 @@ func (bs *BytesString) String() string { } return string(*bs) } + +func mergeMapsRecursive(baseMap, overrideMap map[string]any) { + if len(overrideMap) == 0 { + return + } + for key, overrideValue := range overrideMap { + if baseVal, ok := baseMap[key]; ok { + if baseMapNested, isBaseMap := baseVal.(map[string]any); isBaseMap { + if overrideMapNested, isOverrideMap := overrideValue.(map[string]any); isOverrideMap { + mergeMapsRecursive(baseMapNested, overrideMapNested) + continue + } + } + } + baseMap[key] = overrideValue + } +} + +// unmarshalToMap decodes JSON into a map, preserving exact integer values via json.Number +// instead of rounding them through float64. +func unmarshalToMap(data []byte) (map[string]any, error) { + var m map[string]any + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(&m); err != nil { + return nil, err + } + return m, nil +} + +// MergeDeep merges an override object into a base one using JSON round-trip. +// Fields present in the override overwrite corresponding fields in the base. +// When reverse is true the roles are swapped: base fills absent fields in override +// (useful when the override should win and the base provides defaults). +func MergeDeep[T comparable](base, override T, reverse bool) error { + var zero T + if override == zero { + return nil + } + baseJSON, err := json.Marshal(base) + if err != nil { + return fmt.Errorf("failed to marshal base spec: %w", err) + } + overrideJSON, err := json.Marshal(override) + if err != nil { + return fmt.Errorf("failed to marshal override spec: %w", err) + } + baseMap, err := unmarshalToMap(baseJSON) + if err != nil { + return fmt.Errorf("failed to unmarshal base spec to map: %w", err) + } + overrideMap, err := unmarshalToMap(overrideJSON) + if err != nil { + return fmt.Errorf("failed to unmarshal override spec to map: %w", err) + } + if reverse { + baseMap, overrideMap = overrideMap, baseMap + } + if baseMap == nil { + baseMap = make(map[string]any, len(overrideMap)) + } + mergeMapsRecursive(baseMap, overrideMap) + mergedJSON, err := json.Marshal(baseMap) + if err != nil { + return fmt.Errorf("failed to marshal merged spec map: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(mergedJSON)) + dec.UseNumber() + if err := dec.Decode(base); err != nil { + return fmt.Errorf("failed to unmarshal merged spec JSON: %w", err) + } + return nil +} diff --git a/api/operator/v1beta1/vmextra_types_test.go b/api/operator/v1beta1/vmextra_types_test.go index c90ee705b6..e67a46bca8 100644 --- a/api/operator/v1beta1/vmextra_types_test.go +++ b/api/operator/v1beta1/vmextra_types_test.go @@ -402,3 +402,185 @@ func TestImage_Reference(t *testing.T) { f(Image{Repository: "victoriametrics/vmsingle", Tag: ""}, "victoriametrics/vmsingle:") } + +func TestDeepMerge(t *testing.T) { + type opts struct { + override *VMClusterSpec + reverse bool + validate func(base, merged *VMClusterSpec, err error) + } + f := func(oss ...opts) { + t.Helper() + base := &VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + ClusterVersion: "v1.0.0", + ServiceAccountName: "base", + RetentionPeriod: "30d", + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + ExtraArgs: map[string]string{"keep": "x", "override": "old"}, + }, + }, + VMInsert: &VMInsert{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + ExtraArgs: map[string]string{"insert-arg": "1"}, + }, + }, + }, + } + merged := base.DeepCopy() + for _, o := range oss { + o.validate(base, merged, MergeDeep(merged, o.override, o.reverse)) + } + } + + // with extra args override + f(opts{ + override: &VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + ClusterVersion: "v1.2.3", + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(3)), + ExtraArgs: map[string]string{"override": "new", "add": "y"}, + }, + }, + ServiceAccountName: "zone-sa", + }, + }, + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v1.2.3", merged.ClusterVersion) + assert.Equal(t, "zone-sa", merged.ServiceAccountName) + if !assert.NotNil(t, merged.VMSelect) || !assert.NotNil(t, merged.VMSelect.ReplicaCount) { + return + } + assert.Equal(t, int32(3), *merged.VMSelect.ReplicaCount) + assert.Equal(t, "x", merged.VMSelect.ExtraArgs["keep"]) + assert.Equal(t, "new", merged.VMSelect.ExtraArgs["override"]) + assert.Equal(t, "y", merged.VMSelect.ExtraArgs["add"]) + if !assert.NotNil(t, merged.VMInsert) || !assert.NotNil(t, merged.VMInsert.ReplicaCount) { + return + } + assert.Equal(t, int32(1), *merged.VMInsert.ReplicaCount) + assert.Equal(t, "1", merged.VMInsert.ExtraArgs["insert-arg"]) + }, + }) + + // with nil override spec + f(opts{ + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, base, merged) + }, + }) + + // with empty override spec + f(opts{ + override: &VMClusterSpec{}, + validate: func(base, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, base, merged) + }, + }) + + // with override spec that modifies top-level fields + topLevelOverride := opts{ + override: &VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + ClusterVersion: "v2.0.0", + ServiceAccountName: "global-sa", + }, + }, + validate: func(_, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v2.0.0", merged.ClusterVersion) + assert.Equal(t, "global-sa", merged.ServiceAccountName) + assert.Equal(t, "30d", merged.RetentionPeriod) + }, + } + f(topLevelOverride) + + // multiple overrides applied in sequence + f(topLevelOverride, opts{ + override: &VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + RetentionPeriod: "10d", + ClusterVersion: "v3.0.0", + }, + }, + validate: func(_, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v3.0.0", merged.ClusterVersion) + assert.Equal(t, "global-sa", merged.ServiceAccountName) + assert.Equal(t, "10d", merged.RetentionPeriod) + }, + }) + + // reverse=true (how pool overrides are merged onto the cluster-level base): the struct + // being merged into already has its own fields populated and wins; the "override" argument + // only supplies defaults for fields the first struct leaves absent. + f(opts{ + override: &VMClusterSpec{ + VMClusterSpecBase: VMClusterSpecBase{ + ClusterVersion: "v-defaults", + ServiceAccountName: "sa-defaults", + RetentionPeriod: "90d", + VMSelect: &VMSelect{ + CommonAppsParams: CommonAppsParams{ + ReplicaCount: ptr.To(int32(9)), + ExtraArgs: map[string]string{"from-defaults": "yes"}, + }, + }, + }, + }, + reverse: true, + validate: func(_, merged *VMClusterSpec, err error) { + assert.NoError(t, err) + assert.Equal(t, "v1.0.0", merged.ClusterVersion) + assert.Equal(t, "base", merged.ServiceAccountName) + assert.Equal(t, "30d", merged.RetentionPeriod) + if !assert.NotNil(t, merged.VMSelect) || !assert.NotNil(t, merged.VMSelect.ReplicaCount) { + return + } + assert.Equal(t, int32(1), *merged.VMSelect.ReplicaCount) + assert.Equal(t, "x", merged.VMSelect.ExtraArgs["keep"]) + assert.Equal(t, "yes", merged.VMSelect.ExtraArgs["from-defaults"]) + }, + }) +} + +func TestDeepMerge_NilBase(t *testing.T) { + var base *VMSelect + override := &VMSelect{ + CommonAppsParams: CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + } + assert.NotPanics(t, func() { + assert.Error(t, MergeDeep(base, override, false)) + }) +} + +func TestMergeMapsRecursive(t *testing.T) { + base := map[string]any{ + "a": map[string]any{ + "b": "keep", + "c": "override-me", + }, + "d": "root-keep", + } + override := map[string]any{ + "a": map[string]any{ + "c": "new", + "z": "added", + }, + "e": "root-added", + } + mergeMapsRecursive(base, override) + assert.Equal(t, "keep", base["a"].(map[string]any)["b"]) + assert.Equal(t, "new", base["a"].(map[string]any)["c"]) + assert.Equal(t, "added", base["a"].(map[string]any)["z"]) + assert.Equal(t, "root-keep", base["d"]) + assert.Equal(t, "root-added", base["e"]) +} diff --git a/api/operator/v1beta1/vmuser_types.go b/api/operator/v1beta1/vmuser_types.go index 16ee822f2d..e8879d9c4b 100644 --- a/api/operator/v1beta1/vmuser_types.go +++ b/api/operator/v1beta1/vmuser_types.go @@ -231,6 +231,10 @@ type NamespacedName struct { // (created via spec.serviceSpec) over the default service when building the target URL. // +optional UseExtraService bool `json:"useExtraService,omitempty"` + // Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + // (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + // +optional + Pool string `json:"pool,omitempty"` } // CRDRef describe CRD target reference. @@ -247,7 +251,7 @@ type CRDRef struct { // AsKey returns unique key for object func (cr *CRDRef) AsKey(nsn NamespacedName) string { - return fmt.Sprintf("%s/%s/%s", cr.Kind, nsn.Namespace, nsn.Name) + return fmt.Sprintf("%s/%s/%s/%s", cr.Kind, nsn.Namespace, nsn.Name, nsn.Pool) } // StaticRef - user-defined routing host address. diff --git a/api/operator/v1beta1/zz_generated.deepcopy.go b/api/operator/v1beta1/zz_generated.deepcopy.go index cbd6afb1e9..ec2c973e36 100644 --- a/api/operator/v1beta1/zz_generated.deepcopy.go +++ b/api/operator/v1beta1/zz_generated.deepcopy.go @@ -6213,14 +6213,72 @@ func (in *VMClusterList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMClusterPool) DeepCopyInto(out *VMClusterPool) { + *out = *in + if in.VMStorage != nil { + in, out := &in.VMStorage, &out.VMStorage + *out = new(VMStorage) + (*in).DeepCopyInto(*out) + } + if in.VMInsert != nil { + in, out := &in.VMInsert, &out.VMInsert + *out = new(VMInsert) + (*in).DeepCopyInto(*out) + } + if in.ReplicationFactor != nil { + in, out := &in.ReplicationFactor, &out.ReplicationFactor + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterPool. +func (in *VMClusterPool) DeepCopy() *VMClusterPool { + if in == nil { + return nil + } + out := new(VMClusterPool) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VMClusterSpec) DeepCopyInto(out *VMClusterSpec) { + *out = *in + in.VMClusterSpecBase.DeepCopyInto(&out.VMClusterSpecBase) + if in.Pools != nil { + in, out := &in.Pools, &out.Pools + *out = make([]VMClusterPool, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterSpec. +func (in *VMClusterSpec) DeepCopy() *VMClusterSpec { + if in == nil { + return nil + } + out := new(VMClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMClusterSpecBase) DeepCopyInto(out *VMClusterSpecBase) { *out = *in if in.ReplicationFactor != nil { in, out := &in.ReplicationFactor, &out.ReplicationFactor *out = new(int32) **out = **in } + if in.GlobalReplicationFactor != nil { + in, out := &in.GlobalReplicationFactor, &out.GlobalReplicationFactor + *out = new(int32) + **out = **in + } if in.ImagePullSecrets != nil { in, out := &in.ImagePullSecrets, &out.ImagePullSecrets *out = make([]v1.LocalObjectReference, len(*in)) @@ -6269,12 +6327,12 @@ func (in *VMClusterSpec) DeepCopyInto(out *VMClusterSpec) { } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterSpec. -func (in *VMClusterSpec) DeepCopy() *VMClusterSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMClusterSpecBase. +func (in *VMClusterSpecBase) DeepCopy() *VMClusterSpecBase { if in == nil { return nil } - out := new(VMClusterSpec) + out := new(VMClusterSpecBase) in.DeepCopyInto(out) return out } diff --git a/config/crd/overlay/crd.descriptionless.yaml b/config/crd/overlay/crd.descriptionless.yaml index db5dfddfe9..18b6bc1c81 100644 --- a/config/crd/overlay/crd.descriptionless.yaml +++ b/config/crd/overlay/crd.descriptionless.yaml @@ -4496,6 +4496,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -4503,6 +4505,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -5755,6 +5759,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -5762,6 +5768,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -20774,6 +20782,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -20781,6 +20791,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -22033,6 +22045,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -22040,6 +22054,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -22668,6 +22684,9 @@ spec: type: object type: array type: object + globalReplicationFactor: + format: int32 + type: integer imagePullSecrets: items: properties: @@ -22712,6 +22731,27 @@ spec: type: object paused: type: boolean + pools: + items: + properties: + name: + maxLength: 16 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + replicationFactor: + format: int32 + type: integer + vminsert: + x-kubernetes-preserve-unknown-fields: true + vmstorage: + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map replicationFactor: format: int32 type: integer @@ -25211,6 +25251,9 @@ spec: - retention type: object type: array + retentionPeriod: + pattern: ^[0-9]+(h|d|w|y)?$ + type: string revisionHistoryLimitCount: format: int32 type: integer @@ -25991,6 +26034,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -25998,6 +26043,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -27250,6 +27297,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -27257,6 +27306,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: @@ -43372,6 +43423,8 @@ spec: type: string namespace: type: string + pool: + type: string useExtraService: type: boolean required: @@ -43379,6 +43432,8 @@ spec: - namespace type: object type: array + pool: + type: string useExtraService: type: boolean required: diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index d50469088b..36460bdc14 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -8862,6 +8862,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -8872,6 +8877,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -11450,6 +11460,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -11460,6 +11475,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -41130,6 +41150,11 @@ spec: namespace: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -41140,6 +41165,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -43687,6 +43717,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -43697,6 +43732,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -44841,6 +44881,16 @@ spec: type: object type: array type: object + globalReplicationFactor: + description: |- + GlobalReplicationFactor sets vmselect's -globalReplicationFactor, i.e. how many copies of + every ingested sample are available across distinct storage groups (see Pools). vmselect + continues returning full responses when up to GlobalReplicationFactor-1 groups are + temporarily unavailable. Only meaningful when Pools is set and samples are actually + replicated across pools (e.g. via a remote-write fan-out upstream of the cluster). + See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmstorage-groups-at-vmselect + format: int32 + type: integer imagePullSecrets: description: |- ImagePullSecrets An optional list of references to secrets in the same namespace @@ -44935,6 +44985,57 @@ spec: Paused If set to true all actions on the underlying managed objects are not going to be performed, except for delete actions. type: boolean + pools: + description: |- + Pools defines named groups of vmstorage (and optionally vminsert) components. + Each pool gets its own StatefulSet and headless Service named --. + Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field. + vmselect queries all pools using the pool name as a storage group name (-storageNode=/). + When pools are defined the top-level vmstorage is not deployed; pools replace it entirely. + The top-level vminsert is deployed as a shared insert group across all pools only when no pool + defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. + items: + description: |- + VMClusterPool defines a named group of vmstorage (and optionally vminsert) components + within a VMCluster. Each pool gets its own StatefulSet and headless Service. + properties: + name: + description: |- + Name is the unique identifier for this pool within the cluster. + Used as a suffix for generated resource names (e.g. vmstorage--) and as a + storage group name in vmselect. Kept short since the cluster name itself isn't length-limited, + and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit. + Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. + maxLength: 16 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + replicationFactor: + description: |- + ReplicationFactor overrides the cluster-level replicationFactor for this pool's storage + group at vmselect, rendered as -replicationFactor=:. Pools + without an override use the cluster-level ReplicationFactor, if set. + format: int32 + type: integer + vminsert: + description: |- + VMInsert defines a dedicated vminsert for this pool. + Each field overrides the corresponding field in the top-level vminsert spec. + When nil, the top-level shared vminsert writes to this pool's storage nodes as well. + x-kubernetes-preserve-unknown-fields: true + vmstorage: + description: |- + VMStorage defines pool-specific vmstorage configuration. + Each field overrides the corresponding field in the top-level vmstorage spec. + Fields absent here inherit from the top-level vmstorage. + RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map replicationFactor: description: |- ReplicationFactor defines how many copies of data make among @@ -49965,6 +50066,12 @@ spec: - retention type: object type: array + retentionPeriod: + description: |- + RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance. + Useful when using Pools to implement multi-retention setups. + pattern: ^[0-9]+(h|d|w|y)?$ + type: string revisionHistoryLimitCount: description: |- The number of old ReplicaSets to retain to allow rollback in deployment or @@ -51442,6 +51549,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -51452,6 +51564,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -54030,6 +54147,11 @@ spec: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -54040,6 +54162,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -84576,6 +84703,11 @@ spec: namespace: description: Namespace of the target Kubernetes object type: string + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service @@ -84586,6 +84718,11 @@ spec: - namespace type: object type: array + pool: + description: |- + Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared + (non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. + type: string useExtraService: description: |- UseExtraService instructs the operator to prefer the CR's additional service diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index c0f22c94ab..b9801a7d77 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -17,6 +17,8 @@ aliases: * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VMAnomaly to [v1.30.4](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1304) version * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VT apps to [v0.11.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.11.0) version +* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/): add `spec.pools` support for multi-retention and tiered-storage setups. Each named pool runs isolated `vmstorage` nodes and optionally a dedicated `vminsert`, all queried transparently by `vmselect` as named storage groups. `VMUser`'s `targetRef.pool` field lets a user route to a specific pool's `vminsert`/`vmstorage` instead of the cluster's shared endpoint; referencing an ambiguous or nonexistent endpoint (e.g. the shared `vminsert` once every pool has its own) is now rejected with a clear configuration error instead of silently producing a broken or empty backend URL. See [#741](https://github.com/VictoriaMetrics/operator/issues/741). +* FEATURE: [vmcluster](https://docs.victoriametrics.com/operator/resources/vmcluster/): add `spec.globalReplicationFactor` and per-pool `spec.pools[].replicationFactor`, rendered as `vmselect`'s `-globalReplicationFactor` and per-storage-group `-replicationFactor=:` flags, completing the storage-group support introduced by `spec.pools`. * FEATURE: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): support bounded `spec.reader.workers`, query-level business policies, and `settings.native_threads_per_worker` introduced in vmanomaly v1.30.2. See [#2496](https://github.com/VictoriaMetrics/operator/pull/2496). * FEATURE: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): add typed `spec.writer.batchMaxSeries`, `spec.writer.batchMaxBytes`, and `spec.writer.metricPrefixCacheMaxEntries` fields for the bounded writer controls introduced in [vmanomaly v1.30.3](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1303). See [#2542](https://github.com/VictoriaMetrics/operator/pull/2542). * FEATURE: [vmoperator](https://docs.victoriametrics.com/operator/): Add `k8s_version` label to `vm_app_version` metric. See [#2530](https://github.com/VictoriaMetrics/operator/pull/2530). diff --git a/docs/api.md b/docs/api.md index 5e1002b3fc..afc560ad88 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1726,7 +1726,7 @@ Appears in: [VMDistributedZone (v1alpha1)](#v1alpha1-vmdistributedzone), [VMDist | Field | Description | | --- | --- | | name#
_string_ | _(Optional)_
Name specifies the static name to be used for the new VMCluster. | -| spec#
_[VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec)_ | _(Optional)_
Spec defines the desired state of a new or update spec for existing VMCluster. | +| spec#
_[VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase)_ | _(Optional)_
Spec defines the desired state of a new or update spec for existing VMCluster. | #### VMDistributedZoneCommon {#v1alpha1-vmdistributedzonecommon} @@ -1965,6 +1965,7 @@ Appears in: [TargetRef (v1beta1)](#v1beta1-targetref) | name#
_string_ | _(Required)_
Name of the target Kubernetes object | | namespace#
_string_ | _(Required)_
Namespace of the target Kubernetes object | | objects#
_[NamespacedName (v1beta1)](#v1beta1-namespacedname) array_ | _(Optional)_
Objects defines list of name/namespace pairs that define existing k8s object | +| pool#
_string_ | _(Optional)_
Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared
(non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. | | useExtraService#
_boolean_ | _(Optional)_
UseExtraService instructs the operator to prefer the CR's additional service
(created via spec.serviceSpec) over the default service when building the target URL. | #### Certs {#v1beta1-certs} @@ -2333,7 +2334,7 @@ DownsamplingConfig defines downsampling configuration for VMSingle and VMCluster Requires enterprise license. See https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#downsampling -Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec) +Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec) | Field | Description | | --- | --- | @@ -2970,7 +2971,7 @@ License holds license key for enterprise features. Using license key is supported starting from VictoriaMetrics corev1.94.0. See [here](https://docs.victoriametrics.com/victoriametrics/enterprise/) -Appears in: [VLAgentSpec (v1)](#v1-vlagentspec), [VLClusterSpec (v1)](#v1-vlclusterspec), [VLDistributedSpec (v1alpha1)](#v1alpha1-vldistributedspec), [VLDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vldistributedzoneagentspec), [VLSingleSpec (v1)](#v1-vlsinglespec), [VMAgentSpec (v1beta1)](#v1beta1-vmagentspec), [VMAlertSpec (v1beta1)](#v1beta1-vmalertspec), [VMAnomalySpec (v1)](#v1-vmanomalyspec), [VMAuthLoadBalancerSpec (v1beta1)](#v1beta1-vmauthloadbalancerspec), [VMAuthSpec (v1beta1)](#v1beta1-vmauthspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMDistributedSpec (v1alpha1)](#v1alpha1-vmdistributedspec), [VMDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vmdistributedzoneagentspec), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec) +Appears in: [VLAgentSpec (v1)](#v1-vlagentspec), [VLClusterSpec (v1)](#v1-vlclusterspec), [VLDistributedSpec (v1alpha1)](#v1alpha1-vldistributedspec), [VLDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vldistributedzoneagentspec), [VLSingleSpec (v1)](#v1-vlsinglespec), [VMAgentSpec (v1beta1)](#v1beta1-vmagentspec), [VMAlertSpec (v1beta1)](#v1beta1-vmalertspec), [VMAnomalySpec (v1)](#v1-vmanomalyspec), [VMAuthLoadBalancerSpec (v1beta1)](#v1beta1-vmauthloadbalancerspec), [VMAuthSpec (v1beta1)](#v1beta1-vmauthspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase), [VMDistributedSpec (v1alpha1)](#v1alpha1-vmdistributedspec), [VMDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vmdistributedzoneagentspec), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec) | Field | Description | | --- | --- | @@ -3030,7 +3031,7 @@ Appears in: [Receiver (v1beta1)](#v1beta1-receiver) ManagedObjectsMetadata contains Labels and Annotations -Appears in: [VLAgentSpec (v1)](#v1-vlagentspec), [VLClusterSpec (v1)](#v1-vlclusterspec), [VLDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vldistributedzoneagentspec), [VLSingleSpec (v1)](#v1-vlsinglespec), [VLogsSpec (v1beta1)](#v1beta1-vlogsspec), [VMAgentSpec (v1beta1)](#v1beta1-vmagentspec), [VMAlertSpec (v1beta1)](#v1beta1-vmalertspec), [VMAlertmanagerSpec (v1beta1)](#v1beta1-vmalertmanagerspec), [VMAnomalySpec (v1)](#v1-vmanomalyspec), [VMAuthSpec (v1beta1)](#v1beta1-vmauthspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vmdistributedzoneagentspec), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec), [VMUserSpec (v1beta1)](#v1beta1-vmuserspec), [VTAgentSpec (v1)](#v1-vtagentspec), [VTClusterSpec (v1)](#v1-vtclusterspec), [VTSingleSpec (v1)](#v1-vtsinglespec) +Appears in: [VLAgentSpec (v1)](#v1-vlagentspec), [VLClusterSpec (v1)](#v1-vlclusterspec), [VLDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vldistributedzoneagentspec), [VLSingleSpec (v1)](#v1-vlsinglespec), [VLogsSpec (v1beta1)](#v1beta1-vlogsspec), [VMAgentSpec (v1beta1)](#v1beta1-vmagentspec), [VMAlertSpec (v1beta1)](#v1beta1-vmalertspec), [VMAlertmanagerSpec (v1beta1)](#v1beta1-vmalertmanagerspec), [VMAnomalySpec (v1)](#v1-vmanomalyspec), [VMAuthSpec (v1beta1)](#v1beta1-vmauthspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase), [VMDistributedZoneAgentSpec (v1alpha1)](#v1alpha1-vmdistributedzoneagentspec), [VMSingleSpec (v1beta1)](#v1beta1-vmsinglespec), [VMUserSpec (v1beta1)](#v1beta1-vmuserspec), [VTAgentSpec (v1)](#v1-vtagentspec), [VTClusterSpec (v1)](#v1-vtclusterspec), [VTSingleSpec (v1)](#v1-vtsinglespec) | Field | Description | | --- | --- | @@ -3168,6 +3169,7 @@ Appears in: [CRDRef (v1beta1)](#v1beta1-crdref) | --- | --- | | name#
_string_ | _(Required)_
Name of the target Kubernetes object | | namespace#
_string_ | _(Required)_
Namespace of the target Kubernetes object | +| pool#
_string_ | _(Optional)_
Pool selects a specific named pool from a VMCluster's spec.pools instead of its shared
(non-pool) endpoint. Only valid for VMCluster/vminsert and VMCluster/vmstorage kinds. | | useExtraService#
_boolean_ | _(Optional)_
UseExtraService instructs the operator to prefer the CR's additional service
(created via spec.serviceSpec) over the default service when building the target URL. | #### NomadSDConfig {#v1beta1-nomadsdconfig} @@ -4792,7 +4794,7 @@ VMAuth is the Schema for the vmauths API VMAuthLoadBalancer configures vmauth as a load balancer for the requests -Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VTClusterSpec (v1)](#v1-vtclusterspec) +Appears in: [VLClusterSpec (v1)](#v1-vlclusterspec), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase), [VTClusterSpec (v1)](#v1-vtclusterspec) | Field | Description | | --- | --- | @@ -5039,7 +5041,7 @@ Cluster version with VMClusterDiscovery configures automatic vmstorage node discovery for vminsert and vmselect. It maps to the -storageNode.discoveryInterval and -storageNode.filter flags. -Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMInsert (v1beta1)](#v1beta1-vminsert), [VMSelect (v1beta1)](#v1beta1-vmselect) +Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase), [VMInsert (v1beta1)](#v1beta1-vminsert), [VMSelect (v1beta1)](#v1beta1-vmselect) | Field | Description | | --- | --- | @@ -5047,12 +5049,27 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMInsert (v1beta | filter#
_string_ | _(Optional)_
Filter is an optional regexp filter applied to discovered vmstorage addresses.
Only addresses matching the filter are used; non-matching addresses are ignored. | | interval#
_string_ | _(Optional)_
Interval is the interval for refreshing the vmstorage node list resolved from DNS SRV records.
The minimum supported value is 1s.
Defaults to 2s if not set. | +#### VMClusterPool {#v1beta1-vmclusterpool} + + +VMClusterPool defines a named group of vmstorage (and optionally vminsert) components +within a VMCluster. Each pool gets its own StatefulSet and headless Service. + +Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) + +| Field | Description | +| --- | --- | +| name#
_string_ | _(Required)_
Name is the unique identifier for this pool within the cluster.
Used as a suffix for generated resource names (e.g. vmstorage--) and as a
storage group name in vmselect. Kept short since the cluster name itself isn't length-limited,
and generated StatefulSet/Deployment names must still fit Kubernetes' 63-character limit.
Must be a lowercase alphanumeric DNS label; hyphens allowed in the interior. | +| replicationFactor#
_integer_ | _(Optional)_
ReplicationFactor overrides the cluster-level replicationFactor for this pool's storage
group at vmselect, rendered as -replicationFactor=:. Pools
without an override use the cluster-level ReplicationFactor, if set. | +| vminsert#
_[VMInsert (v1beta1)](#v1beta1-vminsert)_ | _(Optional)_
VMInsert defines a dedicated vminsert for this pool.
Each field overrides the corresponding field in the top-level vminsert spec.
When nil, the top-level shared vminsert writes to this pool's storage nodes as well. | +| vmstorage#
_[VMStorage (v1beta1)](#v1beta1-vmstorage)_ | _(Optional)_
VMStorage defines pool-specific vmstorage configuration.
Each field overrides the corresponding field in the top-level vmstorage spec.
Fields absent here inherit from the top-level vmstorage.
RetentionPeriod on VMStorage overrides the cluster-level retentionPeriod for this pool. | + #### VMClusterSpec {#v1beta1-vmclusterspec} VMClusterSpec defines the desired state of VMCluster -Appears in: [VMCluster (v1beta1)](#v1beta1-vmcluster), [VMDistributedZoneCluster (v1alpha1)](#v1alpha1-vmdistributedzonecluster) +Appears in: [VMCluster (v1beta1)](#v1beta1-vmcluster) | Field | Description | | --- | --- | @@ -5060,10 +5077,12 @@ Appears in: [VMCluster (v1beta1)](#v1beta1-vmcluster), [VMDistributedZoneCluster | clusterVersion#
_string_ | _(Optional)_
ClusterVersion defines default images tag for all components.
it can be overwritten with component specific image.tag value. | | discovery#
_[VMClusterDiscovery (v1beta1)](#v1beta1-vmclusterdiscovery)_ | _(Optional)_
Discovery configures automatic vmstorage node discovery for vminsert and vmselect.
This is an enterprise feature and requires a valid license key.
See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery | | downsampling#
_[DownsamplingConfig (v1beta1)](#v1beta1-downsamplingconfig)_ | _(Optional)_
Downsampling defines downsampling rules applied to vmselect and vmstorage components.
Requires enterprise license. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#downsampling | +| globalReplicationFactor#
_integer_ | _(Optional)_
GlobalReplicationFactor sets vmselect's -globalReplicationFactor, i.e. how many copies of
every ingested sample are available across distinct storage groups (see Pools). vmselect
continues returning full responses when up to GlobalReplicationFactor-1 groups are
temporarily unavailable. Only meaningful when Pools is set and samples are actually
replicated across pools (e.g. via a remote-write fan-out upstream of the cluster).
See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmstorage-groups-at-vmselect | | imagePullSecrets#
_[LocalObjectReference (v1)](#v1-localobjectreference) array_ | _(Optional)_
ImagePullSecrets An optional list of references to secrets in the same namespace
to use for pulling images from registries
see https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod | | license#
_[License (v1beta1)](#v1beta1-license)_ | _(Optional)_
License allows to configure license key to be used for enterprise features.
Using license key is supported starting from VictoriaMetrics v1.94.0.
See [here](https://docs.victoriametrics.com/victoriametrics/enterprise/) | | managedMetadata#
_[ManagedObjectsMetadata (v1beta1)](#v1beta1-managedobjectsmetadata)_ | _(Required)_
ManagedMetadata defines metadata that will be added to the all objects
created by operator for the given CustomResource | | paused#
_boolean_ | _(Optional)_
Paused If set to true all actions on the underlying managed objects are not
going to be performed, except for delete actions. | +| pools#
_[VMClusterPool (v1beta1)](#v1beta1-vmclusterpool) array_ | _(Optional)_
Pools defines named groups of vmstorage (and optionally vminsert) components.
Each pool gets its own StatefulSet and headless Service named --.
Top-level vmstorage and vminsert specs act as defaults; pool specs override them field-by-field.
vmselect queries all pools using the pool name as a storage group name (-storageNode=/).
When pools are defined the top-level vmstorage is not deployed; pools replace it entirely.
The top-level vminsert is deployed as a shared insert group across all pools only when no pool
defines its own vminsert; as soon as any pool has a dedicated vminsert the top-level one is skipped. | | replicationFactor#
_integer_ | _(Optional)_
ReplicationFactor defines how many copies of data make among
distinct storage nodes | | requestsLoadBalancer#
_[VMAuthLoadBalancer (v1beta1)](#v1beta1-vmauthloadbalancer)_ | _(Required)_
RequestsLoadBalancer configures load-balancing for vminsert and vmselect requests.
It helps to evenly spread load across pods.
Usually it's not possible with Kubernetes TCP-based services.
See more [here](https://docs.victoriametrics.com/operator/resources/vmcluster/#requests-load-balancing) | | retentionPeriod#
_string_ | _(Optional)_
RetentionPeriod defines how long to retain stored metrics, specified as a duration (e.g., "1d", "1w", "1m").
Data with timestamps outside the RetentionPeriod is automatically deleted. The minimum allowed value is 1d, or 24h.
The default value is 1 (one month).
See [retention](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) docs for details. | @@ -5074,10 +5093,38 @@ Appears in: [VMCluster (v1beta1)](#v1beta1-vmcluster), [VMDistributedZoneCluster | vmselect#
_[VMSelect (v1beta1)](#v1beta1-vmselect)_ | _(Optional)_
| | vmstorage#
_[VMStorage (v1beta1)](#v1beta1-vmstorage)_ | _(Optional)_
| +#### VMClusterSpecBase {#v1beta1-vmclusterspecbase} + + +VMClusterSpecBase defines the desired state of VMCluster, excluding Pools. + +Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMDistributedZoneCluster (v1alpha1)](#v1alpha1-vmdistributedzonecluster) + +| Field | Description | +| --- | --- | +| clusterDomainName#
_string_ | _(Optional)_
ClusterDomainName defines domain name suffix for in-cluster dns addresses
aka .cluster.local
used by vminsert and vmselect to build vmstorage address | +| clusterVersion#
_string_ | _(Optional)_
ClusterVersion defines default images tag for all components.
it can be overwritten with component specific image.tag value. | +| discovery#
_[VMClusterDiscovery (v1beta1)](#v1beta1-vmclusterdiscovery)_ | _(Optional)_
Discovery configures automatic vmstorage node discovery for vminsert and vmselect.
This is an enterprise feature and requires a valid license key.
See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#automatic-vmstorage-discovery | +| downsampling#
_[DownsamplingConfig (v1beta1)](#v1beta1-downsamplingconfig)_ | _(Optional)_
Downsampling defines downsampling rules applied to vmselect and vmstorage components.
Requires enterprise license. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#downsampling | +| globalReplicationFactor#
_integer_ | _(Optional)_
GlobalReplicationFactor sets vmselect's -globalReplicationFactor, i.e. how many copies of
every ingested sample are available across distinct storage groups (see Pools). vmselect
continues returning full responses when up to GlobalReplicationFactor-1 groups are
temporarily unavailable. Only meaningful when Pools is set and samples are actually
replicated across pools (e.g. via a remote-write fan-out upstream of the cluster).
See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#vmstorage-groups-at-vmselect | +| imagePullSecrets#
_[LocalObjectReference (v1)](#v1-localobjectreference) array_ | _(Optional)_
ImagePullSecrets An optional list of references to secrets in the same namespace
to use for pulling images from registries
see https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod | +| license#
_[License (v1beta1)](#v1beta1-license)_ | _(Optional)_
License allows to configure license key to be used for enterprise features.
Using license key is supported starting from VictoriaMetrics v1.94.0.
See [here](https://docs.victoriametrics.com/victoriametrics/enterprise/) | +| managedMetadata#
_[ManagedObjectsMetadata (v1beta1)](#v1beta1-managedobjectsmetadata)_ | _(Required)_
ManagedMetadata defines metadata that will be added to the all objects
created by operator for the given CustomResource | +| paused#
_boolean_ | _(Optional)_
Paused If set to true all actions on the underlying managed objects are not
going to be performed, except for delete actions. | +| replicationFactor#
_integer_ | _(Optional)_
ReplicationFactor defines how many copies of data make among
distinct storage nodes | +| requestsLoadBalancer#
_[VMAuthLoadBalancer (v1beta1)](#v1beta1-vmauthloadbalancer)_ | _(Required)_
RequestsLoadBalancer configures load-balancing for vminsert and vmselect requests.
It helps to evenly spread load across pods.
Usually it's not possible with Kubernetes TCP-based services.
See more [here](https://docs.victoriametrics.com/operator/resources/vmcluster/#requests-load-balancing) | +| retentionPeriod#
_string_ | _(Optional)_
RetentionPeriod defines how long to retain stored metrics, specified as a duration (e.g., "1d", "1w", "1m").
Data with timestamps outside the RetentionPeriod is automatically deleted. The minimum allowed value is 1d, or 24h.
The default value is 1 (one month).
See [retention](https://docs.victoriametrics.com/victoriametrics/single-server-victoriametrics/#retention) docs for details. | +| serviceAccountName#
_string_ | _(Optional)_
ServiceAccountName is the name of the ServiceAccount to use to run the
VMSelect, VMStorage and VMInsert Pods. | +| useLegacyNaming#
_boolean_ | _(Optional)_
UseLegacyNaming switches component resource naming from the default prefix convention
(-, e.g. "vmselect-myapp") to a suffix convention (-,
e.g. "myapp-vmselect"). Useful when migrating from standalone Helm charts.
Available from: v0.73.0 | +| useStrictSecurity#
_boolean_ | _(Optional)_
UseStrictSecurity enables strict security mode for component
it restricts disk writes access
uses non-root user out of the box
drops not needed security permissions | +| vminsert#
_[VMInsert (v1beta1)](#v1beta1-vminsert)_ | _(Optional)_
| +| vmselect#
_[VMSelect (v1beta1)](#v1beta1-vmselect)_ | _(Optional)_
| +| vmstorage#
_[VMStorage (v1beta1)](#v1beta1-vmstorage)_ | _(Optional)_
| + #### VMInsert {#v1beta1-vminsert} -Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) +Appears in: [VMClusterPool (v1beta1)](#v1beta1-vmclusterpool), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase) | Field | Description | | --- | --- | @@ -5463,7 +5510,7 @@ Appears in: [Endpoint (v1beta1)](#v1beta1-endpoint), [EndpointScrapeParams (v1be VMSelect defines configuration section for vmselect components of the victoria-metrics cluster -Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) +Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase) | Field | Description | | --- | --- | @@ -5727,7 +5774,7 @@ Appears in: [VMStaticScrape (v1beta1)](#v1beta1-vmstaticscrape) #### VMStorage {#v1beta1-vmstorage} -Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) +Appears in: [VMClusterPool (v1beta1)](#v1beta1-vmclusterpool), [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec), [VMClusterSpecBase (v1beta1)](#v1beta1-vmclusterspecbase) | Field | Description | | --- | --- | @@ -5771,6 +5818,7 @@ Appears in: [VMClusterSpec (v1beta1)](#v1beta1-vmclusterspec) | replicaCount#
_integer_ | _(Optional)_
ReplicaCount is the expected size of the Application. | | resources#
_[ResourceRequirements (v1)](#v1-resourcerequirements)_ | _(Optional)_
Resources container resource request and limits, https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
if not defined default resources from operator config will be used | | retentionFilters#
_[RetentionFiltersConfig (v1beta1)](#v1beta1-retentionfiltersconfig)_ | _(Optional)_
RetentionFilters defines per-series retention filters for vmstorage.
Requires enterprise license. See https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#retention-filters | +| retentionPeriod#
_string_ | _(Optional)_
RetentionPeriod overrides the cluster-level retentionPeriod for this storage instance.
Useful when using Pools to implement multi-retention setups. | | revisionHistoryLimitCount#
_integer_ | _(Optional)_
The number of old ReplicaSets to retain to allow rollback in deployment or
maximum number of revisions that will be maintained in the Deployment revision history.
Has no effect at StatefulSets
Defaults to 10. | | rollingUpdateStrategy#
_[StatefulSetUpdateStrategyType (v1)](#v1-statefulsetupdatestrategytype)_ | _(Optional)_
RollingUpdateStrategy defines strategy for application updates
Default is OnDelete, in this case operator handles update process
Can be changed for RollingUpdate | | rollingUpdateStrategyBehavior#
_[StatefulSetUpdateStrategyBehavior (v1beta1)](#v1beta1-statefulsetupdatestrategybehavior)_ | _(Optional)_
RollingUpdateStrategyBehavior defines customized behavior for rolling updates.
It applies if the RollingUpdateStrategy is set to OnDelete, which is the default. | diff --git a/docs/resources/vmcluster.md b/docs/resources/vmcluster.md index 4d9c08e4ef..44840a8a7a 100644 --- a/docs/resources/vmcluster.md +++ b/docs/resources/vmcluster.md @@ -867,6 +867,157 @@ spec: enabled: false ``` +## Storage pools + +`spec.pools` allows you to run multiple isolated groups of `vmstorage` nodes within a single `VMCluster`. +Each pool has its own name, StatefulSet, headless Service, and optional PodDisruptionBudget / HPA / VPA. +`vmselect` queries all pools simultaneously and merges results transparently, so existing queries require no changes. + +The primary use case is **multi-retention**: assign a different `retentionPeriod` to each pool so that some metrics expire sooner than others, without deploying separate clusters. + +When `spec.pools` is defined the top-level `spec.vmstorage` is **not** deployed; pools replace it entirely. + +### Pool configuration and inheritance + +Each pool entry has a `name` and an optional `vmstorage` override: + +| Field | Description | +|-------|-------------| +| `name` | Unique identifier within the cluster. Used as a resource name suffix and as the storage group name in `vmselect`. Must match `^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`, max 16 chars. | +| `vmstorage` | Pool-specific `vmstorage` configuration. Each field overrides the corresponding field in the top-level `spec.vmstorage`; absent fields inherit from the top-level. | +| `vminsert` | Optional dedicated `vminsert` for this pool. When nil and no pool defines a dedicated `vminsert`, the shared top-level `vminsert` writes to this pool's storage nodes as well. | + +`vmstorage.retentionPeriod` inside a pool overrides `spec.retentionPeriod` for that pool, enabling per-pool retention without enterprise retention filters. + +### Resource naming + +Pool resources follow the pattern `vm--`: + +| Resource | Example | +|----------|---------| +| `vmstorage` StatefulSet | `vmstorage-mycluster-hot` | +| `vmstorage` headless Service | `vmstorage-mycluster-hot` | +| `vminsert` Deployment (dedicated) | `vminsert-mycluster-hot` | + +All pool pods carry an extra label `app.kubernetes.io/pool=` to keep per-pool selectors disjoint. + +### Shared-insert example + +Two pools with different retention periods sharing a single `vminsert` and `vmselect`: + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: example-pools +spec: + retentionPeriod: "30d" # default, used as fallback + vmselect: + replicaCount: 2 + vminsert: + replicaCount: 2 # shared across both pools + # top-level vmstorage sets shared defaults inherited by all pools + vmstorage: + replicaCount: 3 + storage: + volumeClaimTemplate: + spec: + resources: + requests: + storage: 50Gi + pools: + - name: hot + vmstorage: + retentionPeriod: "7d" + - name: cold + vmstorage: + replicaCount: 5 + retentionPeriod: "365d" + storage: + volumeClaimTemplate: + spec: + resources: + requests: + storage: 2Ti +``` + +This creates: +- `vmstorage-example-pools-hot` StatefulSet with 3 replicas (inherited) and 7-day retention. +- `vmstorage-example-pools-cold` StatefulSet with 5 replicas (overridden) and 365-day retention. +- One shared `vminsert-example-pools` Deployment writing to both pools. +- `vmselect` queries both pools and returns merged results. + +With a shared `vminsert`, incoming series are distributed across the combined set of `hot` and +`cold` storage nodes without regard to retention: any given series can land on either pool, so a +series is not guaranteed to expire after `hot`'s 7-day retention — it may just as well have been +routed to `cold` and stay queryable for 365 days. A shared `vminsert` is only useful for scaling +storage capacity/replica count across pools; deliberately controlling **which** series get which +retention requires giving each pool its own `vminsert` and routing writes to the desired pool +explicitly (see below). + +### Pool with a dedicated vminsert + +When a pool defines `vminsert`, it gets its own isolated insert path. +The shared top-level `vminsert` is deployed only when **no** pool defines its own `vminsert`; as soon as any pool has a dedicated insert, the shared one is skipped, so every pool must define its own `vminsert` in that case. +Use this when pools need different write-path configurations (e.g. different stream aggregation rules or resource limits): + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMCluster +metadata: + name: example-dedicated-insert +spec: + retentionPeriod: "30d" + vmselect: + replicaCount: 2 + # top-level vminsert sets shared defaults inherited by pool-level inserts + vminsert: + replicaCount: 2 + resources: + limits: + cpu: "1" + memory: 512Mi + vmstorage: + replicaCount: 3 + pools: + - name: short + vmstorage: + retentionPeriod: "7d" + vminsert: + replicaCount: 4 # override: more replicas for high-throughput writes + - name: long + vmstorage: + retentionPeriod: "2y" + vminsert: {} # dedicated insert inheriting all top-level vminsert defaults +``` + +Each pool's dedicated `vminsert` gets its own Service, named `vminsert--` +(e.g. `vminsert-example-dedicated-insert-short`). A writer picks a pool by targeting that +Service directly, e.g. from `vmagent`: + +```yaml +remoteWrite: + - url: http://vminsert-example-dedicated-insert-short.default.svc:8480/insert/multitenant/prometheus/api/v1/write +``` + +Or, when writes go through `VMAuth`, a `VMUser` can route to a specific pool with +`targetRef.pool` instead of a raw URL: + +```yaml +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMUser +metadata: + name: writer-short +spec: + targetRefs: + - crd: + kind: VMCluster/vminsert + name: example-dedicated-insert + namespace: default + pool: short + paths: ["/insert/.*"] +``` + ## Examples ### Minimal example without persistence diff --git a/internal/controller/operator/factory/build/build.go b/internal/controller/operator/factory/build/build.go index 6ccc17ee03..40506fc47c 100644 --- a/internal/controller/operator/factory/build/build.go +++ b/internal/controller/operator/factory/build/build.go @@ -3,7 +3,6 @@ package build import ( "bytes" "compress/gzip" - "encoding/json" "fmt" "io" "path" @@ -318,68 +317,3 @@ func GunzipConfig(data []byte) ([]byte, error) { defer gr.Close() return io.ReadAll(gr) } - -// mergeMapsRecursive deeply merges overrideMap into baseMap. -// It handles nested maps (which correspond to nested structs after JSON unmarshal). -// Values from overrideMap overwrite values in baseMap. -func mergeMapsRecursive(baseMap, overrideMap map[string]any) { - if len(overrideMap) == 0 { - return - } - for key, overrideValue := range overrideMap { - if baseVal, ok := baseMap[key]; ok { - if baseMapNested, isBaseMap := baseVal.(map[string]any); isBaseMap { - if overrideMapNested, isOverrideMap := overrideValue.(map[string]any); isOverrideMap { - // Both are nested maps, recurse - mergeMapsRecursive(baseMapNested, overrideMapNested) - continue - } - } - } - baseMap[key] = overrideValue - } -} - -// MergeDeep merges an override object into a base one. -// Fields present in the override will overwrite corresponding fields in the base. -func MergeDeep[T comparable](base, override T, reverse bool) error { - var zero T - if override == zero { - return nil - } - - baseJSON, err := json.Marshal(base) - if err != nil { - return fmt.Errorf("failed to marshal base spec: %w", err) - } - overrideJSON, err := json.Marshal(override) - if err != nil { - return fmt.Errorf("failed to marshal override spec: %w", err) - } - - var baseMap map[string]any - if err := json.Unmarshal(baseJSON, &baseMap); err != nil { - return fmt.Errorf("failed to unmarshal base spec to map: %w", err) - } - var overrideMap map[string]any - if err := json.Unmarshal(overrideJSON, &overrideMap); err != nil { - return fmt.Errorf("failed to unmarshal override spec to map: %w", err) - } - - // Perform a deep merge: fields from overrideMap recursively overwrite corresponding fields in baseMap. - // If an override value is explicitly nil, it signifies the removal or nullification of that field. - if reverse { - baseMap, overrideMap = overrideMap, baseMap - } - mergeMapsRecursive(baseMap, overrideMap) - mergedSpecJSON, err := json.Marshal(baseMap) - if err != nil { - return fmt.Errorf("failed to marshal merged spec map: %w", err) - } - - if err := json.Unmarshal(mergedSpecJSON, base); err != nil { - return fmt.Errorf("failed to unmarshal merged spec JSON: %w", err) - } - - return nil -} diff --git a/internal/controller/operator/factory/build/build_test.go b/internal/controller/operator/factory/build/build_test.go index 1241c7a7d2..ddee8e2ae5 100644 --- a/internal/controller/operator/factory/build/build_test.go +++ b/internal/controller/operator/factory/build/build_test.go @@ -188,150 +188,3 @@ func TestGzipGunzipConfig(t *testing.T) { // binary data f([]int32{1, 2, 3, 4}) } - -func TestMergeMapsRecursive(t *testing.T) { - base := map[string]any{ - "a": map[string]any{ - "b": "keep", - "c": "override-me", - }, - "d": "root-keep", - } - override := map[string]any{ - "a": map[string]any{ - "c": "new", - "z": "added", - }, - "e": "root-added", - } - // initial merge - mergeMapsRecursive(base, override) - assert.Equal(t, "keep", base["a"].(map[string]any)["b"]) - assert.Equal(t, "new", base["a"].(map[string]any)["c"]) - assert.Equal(t, "added", base["a"].(map[string]any)["z"]) - assert.Equal(t, "root-keep", base["d"]) - assert.Equal(t, "root-added", base["e"]) -} - -func TestDeepMerge(t *testing.T) { - type opts struct { - override *vmv1beta1.VMClusterSpec - validate func(base, merged *vmv1beta1.VMClusterSpec, err error) - } - f := func(oss ...opts) { - t.Helper() - base := &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v1.0.0", - ServiceAccountName: "base", - RetentionPeriod: "30d", - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - ExtraArgs: map[string]string{"keep": "x", "override": "old"}, - }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - ExtraArgs: map[string]string{"insert-arg": "1"}, - }, - }, - } - merged := base.DeepCopy() - for _, o := range oss { - o.validate(base, merged, MergeDeep(merged, o.override, false)) - } - } - - // with extra args override - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v1.2.3", - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(3)), - ExtraArgs: map[string]string{"override": "new", "add": "y"}, - }, - }, - ServiceAccountName: "zone-sa", - }, - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - - // top-level - assert.Equal(t, "v1.2.3", merged.ClusterVersion) - assert.Equal(t, "zone-sa", merged.ServiceAccountName) - - // nested merge - if !assert.NotNil(t, merged.VMSelect) || !assert.NotNil(t, merged.VMSelect.ReplicaCount) { - return - } - assert.Equal(t, int32(3), *merged.VMSelect.ReplicaCount) - assert.Equal(t, "x", merged.VMSelect.ExtraArgs["keep"]) - assert.Equal(t, "new", merged.VMSelect.ExtraArgs["override"]) - assert.Equal(t, "y", merged.VMSelect.ExtraArgs["add"]) - - // untouched subtree - if !assert.NotNil(t, merged.VMInsert) || !assert.NotNil(t, merged.VMInsert.ReplicaCount) { - return - } - assert.Equal(t, int32(1), *merged.VMInsert.ReplicaCount) - assert.Equal(t, "1", merged.VMInsert.ExtraArgs["insert-arg"]) - }, - }) - - // with nil override spec - f(opts{ - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, base, merged) - }, - }) - - // with empty override spec - f(opts{ - override: &vmv1beta1.VMClusterSpec{}, - validate: func(base, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, base, merged) - }, - }) - - // with override spec that modifies top-level fields - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v2.0.0", - ServiceAccountName: "global-sa", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.Equal(t, "v2.0.0", merged.ClusterVersion) - assert.Equal(t, "global-sa", merged.ServiceAccountName) - assert.Equal(t, "30d", merged.RetentionPeriod) - }, - }) - - // multiple overrides - f(opts{ - override: &vmv1beta1.VMClusterSpec{ - ClusterVersion: "v2.0.0", - ServiceAccountName: "global-sa", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, "v2.0.0", merged.ClusterVersion) - assert.Equal(t, "global-sa", merged.ServiceAccountName) - assert.Equal(t, "30d", merged.RetentionPeriod) - }, - }, opts{ - override: &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "10d", - ClusterVersion: "v3.0.0", - }, - validate: func(_, merged *vmv1beta1.VMClusterSpec, err error) { - assert.NoError(t, err) - assert.Equal(t, "v3.0.0", merged.ClusterVersion) // Cluster-specific override should take precedence - assert.Equal(t, "global-sa", merged.ServiceAccountName) // From global override, unchanged by cluster override - assert.Equal(t, "10d", merged.RetentionPeriod) // From cluster override - }, - }) -} diff --git a/internal/controller/operator/factory/build/cluster.go b/internal/controller/operator/factory/build/cluster.go index 76e917979b..bd6c8a0637 100644 --- a/internal/controller/operator/factory/build/cluster.go +++ b/internal/controller/operator/factory/build/cluster.go @@ -25,11 +25,12 @@ type ChildBuilder struct { kind vmv1beta1.ClusterComponent finalLabels map[string]string selectorLabels map[string]string + prefixedName string } // PrefixedName implements build.svcBuilderArgs interface func (b *ChildBuilder) PrefixedName() string { - return b.ParentOpts.PrefixedName(b.kind) + return b.prefixedName } // FinalLabels implements build.svcBuilderArgs interface @@ -55,11 +56,30 @@ func (b *ChildBuilder) SetSelectorLabels(ls map[string]string) { b.selectorLabels = ls } +// poolLabelKey is the label added to all pool-scoped resources. +const poolLabelKey = "app.kubernetes.io/pool" + func NewChildBuilder(cr ParentOpts, kind vmv1beta1.ClusterComponent) *ChildBuilder { + return NewPoolBuilder(cr, kind, "") +} + +// NewPoolBuilder is like NewChildBuilder but scopes the builder to a named pool. +// PrefixedName appends "-", and both SelectorLabels and FinalLabels include +// the pool label so that multiple pools in the same namespace have non-overlapping selectors. +func NewPoolBuilder(cr ParentOpts, kind vmv1beta1.ClusterComponent, poolName string) *ChildBuilder { + selectorLabels := cr.SelectorLabels(kind) + finalLabels := cr.FinalLabels(kind) + name := cr.PrefixedName(kind) + if poolName != "" { + name += "-" + poolName + selectorLabels[poolLabelKey] = poolName + finalLabels[poolLabelKey] = poolName + } return &ChildBuilder{ ParentOpts: cr, kind: kind, - finalLabels: cr.FinalLabels(kind), - selectorLabels: cr.SelectorLabels(kind), + finalLabels: finalLabels, + selectorLabels: selectorLabels, + prefixedName: name, } } diff --git a/internal/controller/operator/factory/build/defaults.go b/internal/controller/operator/factory/build/defaults.go index 4e92a9ed35..1aab15ea87 100644 --- a/internal/controller/operator/factory/build/defaults.go +++ b/internal/controller/operator/factory/build/defaults.go @@ -501,6 +501,65 @@ func addRequestsLoadBalancerDefaults(lb *vmv1beta1.VMAuthLoadBalancer, cp *commo } } +func addVMClusterStorageDefaults(vms *vmv1beta1.VMStorage, c *config.BaseOperatorConf, cp commonParams) { + if vms == nil { + return + } + if vms.StorageDataPath == "" { + vms.StorageDataPath = vmStorageDefaultDBPath + } + if vms.VMInsertPort == "" { + vms.VMInsertPort = c.VMCluster.Storage.VMInsertPort + } + if vms.VMSelectPort == "" { + vms.VMSelectPort = c.VMCluster.Storage.VMSelectPort + } + cv := config.ApplicationDefaults(c.VMCluster.Storage.Common) + cpStorage := cp + cpStorage.tag = setTag(vms.ComponentVersion, cp.tag) + addDefaultsToCommonParams(&vms.CommonAppsParams, &cpStorage, &cv) + vms.PreStopSleepSeconds = nil + + bv := config.ApplicationDefaults(c.VMBackup) + useBackupDefaultResources := c.VMBackup.UseDefaultResources + if vms.UseDefaultResources != nil { + useBackupDefaultResources = *vms.UseDefaultResources + } + addDefaultsToVMBackup(vms.VMBackup, useBackupDefaultResources, &bv) +} + +func addVMClusterInsertDefaults(vmi *vmv1beta1.VMInsert, c *config.BaseOperatorConf, cp commonParams) { + if vmi == nil { + return + } + cv := config.ApplicationDefaults(c.VMCluster.Insert) + cpInsert := cp + cpInsert.tag = setTag(vmi.ComponentVersion, cp.tag) + addDefaultsToCommonParams(&vmi.CommonAppsParams, &cpInsert, &cv) +} + +func DefaultPoolVMStorage(vms *vmv1beta1.VMStorage, cr *vmv1beta1.VMCluster) { + c := getCfg() + cp := commonParams{ + useStrictSecurity: cr.Spec.UseStrictSecurity, + tag: cr.Spec.ClusterVersion, + license: cr.Spec.License, + imagePullSecrets: cr.Spec.ImagePullSecrets, + } + addVMClusterStorageDefaults(vms, c, cp) +} + +func DefaultPoolVMInsert(vmi *vmv1beta1.VMInsert, cr *vmv1beta1.VMCluster) { + c := getCfg() + cp := commonParams{ + useStrictSecurity: cr.Spec.UseStrictSecurity, + tag: cr.Spec.ClusterVersion, + license: cr.Spec.License, + imagePullSecrets: cr.Spec.ImagePullSecrets, + } + addVMClusterInsertDefaults(vmi, c, cp) +} + func addVMClusterDefaults(objI any) { cr := objI.(*vmv1beta1.VMCluster) c := getCfg() @@ -514,35 +573,8 @@ func addVMClusterDefaults(objI any) { license: cr.Spec.License, imagePullSecrets: cr.Spec.ImagePullSecrets, } - if cr.Spec.VMStorage != nil { - if cr.Spec.VMStorage.StorageDataPath == "" { - cr.Spec.VMStorage.StorageDataPath = vmStorageDefaultDBPath - } - if cr.Spec.VMStorage.VMInsertPort == "" { - cr.Spec.VMStorage.VMInsertPort = c.VMCluster.Storage.VMInsertPort - } - if cr.Spec.VMStorage.VMSelectPort == "" { - cr.Spec.VMStorage.VMSelectPort = c.VMCluster.Storage.VMSelectPort - } - cv := config.ApplicationDefaults(c.VMCluster.Storage.Common) - cpStorage := cp - cpStorage.tag = setTag(cr.Spec.VMStorage.ComponentVersion, cp.tag) - addDefaultsToCommonParams(&cr.Spec.VMStorage.CommonAppsParams, &cpStorage, &cv) - cr.Spec.VMStorage.PreStopSleepSeconds = nil - - bv := config.ApplicationDefaults(c.VMBackup) - useBackupDefaultResources := c.VMBackup.UseDefaultResources - if cr.Spec.VMStorage.UseDefaultResources != nil { - useBackupDefaultResources = *cr.Spec.VMStorage.UseDefaultResources - } - addDefaultsToVMBackup(cr.Spec.VMStorage.VMBackup, useBackupDefaultResources, &bv) - } - if cr.Spec.VMInsert != nil { - cv := config.ApplicationDefaults(c.VMCluster.Insert) - cpInsert := cp - cpInsert.tag = setTag(cr.Spec.VMInsert.ComponentVersion, cp.tag) - addDefaultsToCommonParams(&cr.Spec.VMInsert.CommonAppsParams, &cpInsert, &cv) - } + addVMClusterStorageDefaults(cr.Spec.VMStorage, c, cp) + addVMClusterInsertDefaults(cr.Spec.VMInsert, c, cp) if cr.Spec.VMSelect != nil { if cr.Spec.VMSelect.CacheMountPath == "" { cr.Spec.VMSelect.CacheMountPath = "/cache" diff --git a/internal/controller/operator/factory/build/defaults_test.go b/internal/controller/operator/factory/build/defaults_test.go index cea3727a65..00936b89d1 100644 --- a/internal/controller/operator/factory/build/defaults_test.go +++ b/internal/controller/operator/factory/build/defaults_test.go @@ -257,12 +257,14 @@ func TestClusterComponentVersionDefaults(t *testing.T) { "VMCluster/VMSelect": func(o opts) string { cr := &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - ClusterVersion: o.clusterVersion, - VMSelect: &vmv1beta1.VMSelect{ - ComponentVersion: o.componentVersion, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Image: vmv1beta1.Image{ - Tag: o.imageTag, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ClusterVersion: o.clusterVersion, + VMSelect: &vmv1beta1.VMSelect{ + ComponentVersion: o.componentVersion, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Image: vmv1beta1.Image{ + Tag: o.imageTag, + }, }, }, }, @@ -308,12 +310,14 @@ func TestClusterComponentVersionDefaults(t *testing.T) { "VMCluster/VMInsert": func(o opts) string { cr := &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - ClusterVersion: o.clusterVersion, - VMInsert: &vmv1beta1.VMInsert{ - ComponentVersion: o.componentVersion, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Image: vmv1beta1.Image{ - Tag: o.imageTag, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ClusterVersion: o.clusterVersion, + VMInsert: &vmv1beta1.VMInsert{ + ComponentVersion: o.componentVersion, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Image: vmv1beta1.Image{ + Tag: o.imageTag, + }, }, }, }, @@ -325,12 +329,14 @@ func TestClusterComponentVersionDefaults(t *testing.T) { "VMCluster/VMStorage": func(o opts) string { cr := &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - ClusterVersion: o.clusterVersion, - VMStorage: &vmv1beta1.VMStorage{ - ComponentVersion: o.componentVersion, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Image: vmv1beta1.Image{ - Tag: o.imageTag, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ClusterVersion: o.clusterVersion, + VMStorage: &vmv1beta1.VMStorage{ + ComponentVersion: o.componentVersion, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Image: vmv1beta1.Image{ + Tag: o.imageTag, + }, }, }, }, @@ -453,14 +459,16 @@ func TestClusterComponentVersionDefaults(t *testing.T) { "VMCluster/RequestsLoadBalancer": func(o opts) string { cr := &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - ClusterVersion: o.clusterVersion, - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - Spec: vmv1beta1.VMAuthLoadBalancerSpec{ - ComponentVersion: o.componentVersion, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Image: vmv1beta1.Image{ - Tag: o.imageTag, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ClusterVersion: o.clusterVersion, + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + ComponentVersion: o.componentVersion, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Image: vmv1beta1.Image{ + Tag: o.imageTag, + }, }, }, }, diff --git a/internal/controller/operator/factory/build/security.go b/internal/controller/operator/factory/build/security.go index 6a1d31c103..211256c8a4 100644 --- a/internal/controller/operator/factory/build/security.go +++ b/internal/controller/operator/factory/build/security.go @@ -1,6 +1,9 @@ package build import ( + "errors" + "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/utils/ptr" @@ -61,8 +64,8 @@ func IsOpenShiftCompatibilityActive() bool { } // WarnOpenShiftClusterSpec returns OpenShift security context warnings for all sub-components -// of a VMClusterSpec (VMSelect, VMInsert, VMStorage, RequestsLoadBalancer). -func WarnOpenShiftClusterSpec(spec *vmv1beta1.VMClusterSpec) []string { +// of a VMClusterSpecBase (VMSelect, VMInsert, VMStorage, RequestsLoadBalancer). +func WarnOpenShiftClusterSpec(spec *vmv1beta1.VMClusterSpecBase) []string { var w []string if c := spec.VMSelect; c != nil { w = append(w, WarnOpenShiftSecurityContext(c.SecurityContext)...) @@ -79,6 +82,29 @@ func WarnOpenShiftClusterSpec(spec *vmv1beta1.VMClusterSpec) []string { return w } +// WarnOpenShiftClusterPools returns OpenShift security context warnings for each pool's +// resolved vmstorage and vminsert. +func WarnOpenShiftClusterPools(cr *vmv1beta1.VMCluster) ([]string, error) { + var w []string + var errs []error + for i := range cr.Spec.Pools { + pool := &cr.Spec.Pools[i] + storage, err := cr.ResolvePoolVMStorage(pool) + if err != nil { + errs = append(errs, fmt.Errorf("pool %q: cannot resolve vmstorage: %w", pool.Name, err)) + } else if storage != nil { + w = append(w, WarnOpenShiftSecurityContext(storage.SecurityContext)...) + } + insert, err := cr.ResolvePoolVMInsert(pool) + if err != nil { + errs = append(errs, fmt.Errorf("pool %q: cannot resolve vminsert: %w", pool.Name, err)) + } else if insert != nil { + w = append(w, WarnOpenShiftSecurityContext(insert.SecurityContext)...) + } + } + return w, errors.Join(errs...) +} + // WarnOpenShiftVLClusterSpec returns OpenShift security context warnings for all sub-components // of a VLClusterSpec (VLInsert, VLSelect, VLStorage, RequestsLoadBalancer). func WarnOpenShiftVLClusterSpec(spec *vmv1.VLClusterSpec) []string { diff --git a/internal/controller/operator/factory/reconcile/vmcluster_test.go b/internal/controller/operator/factory/reconcile/vmcluster_test.go index f63dd354ce..0014b014b5 100644 --- a/internal/controller/operator/factory/reconcile/vmcluster_test.go +++ b/internal/controller/operator/factory/reconcile/vmcluster_test.go @@ -27,7 +27,9 @@ func TestVMClusterReconcile(t *testing.T) { Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1d", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1d", + }, }, } for _, fn := range fns { diff --git a/internal/controller/operator/factory/vmauth/vmusers_config.go b/internal/controller/operator/factory/vmauth/vmusers_config.go index 2bc7dbdec5..109c27df8f 100644 --- a/internal/controller/operator/factory/vmauth/vmusers_config.go +++ b/internal/controller/operator/factory/vmauth/vmusers_config.go @@ -47,12 +47,18 @@ func updateCRDObjURLs(ctx context.Context, rclient client.Client, crd *vmv1beta1 if _, ok := objURLs[key]; ok { continue } - crdObj, ok := crdNameToObject[crd.Kind] + newCrdObj, ok := crdNameToObject[crd.Kind] if !ok { return fmt.Errorf("unsupported kind=%q", crd.Kind) } + crdObj := newCrdObj() crdObj.SetName(nsn.Name) crdObj.SetNamespace(nsn.Namespace) + if cw, ok := crdObj.(*clusterWithURL); ok { + cw.pool = nsn.Pool + } else if nsn.Pool != "" { + return fmt.Errorf("pool=%q is not supported for kind=%q", nsn.Pool, crd.Kind) + } url, err := getAsURLObject(ctx, rclient, crdObj, nsn.UseExtraService) if err != nil { if !build.IsNotFound(err) { @@ -79,7 +85,7 @@ func (pos *parsedObjects) buildConfig(ctx context.Context, rclient client.Client backends[ref.Name] = ref } else { if r, ok := backends[ref.Name]; ok { - if err := build.MergeDeep(ref, r, true); err != nil { + if err := vmv1beta1.MergeDeep(ref, r, true); err != nil { return fmt.Errorf("failed to merge target refs: %w", err) } } else { @@ -176,6 +182,10 @@ type objectWithURL interface { AsURL(isExtra bool) string } +type urlErrorer interface { + URLError() error +} + func getAsURLObject(ctx context.Context, rclient client.Client, objT objectWithURL, isExtra bool) (string, error) { obj := objT.(client.Object) // dirty hack to restore original type of vmcluster or vlcluster @@ -191,7 +201,13 @@ func getAsURLObject(ctx context.Context, rclient client.Client, objT objectWithU } return "", fmt.Errorf("cannot get object by given ref namespace=%q,name=%q: %w", obj.GetNamespace(), obj.GetName(), err) } - return objT.AsURL(isExtra), nil + url := objT.AsURL(isExtra) + if ue, ok := objT.(urlErrorer); ok { + if err := ue.URLError(); err != nil { + return "", err + } + } + return url, nil } func (pos *parsedObjects) addAuthCredentialsBuildSecrets(ac *build.AssetsCache) (needToCreateSecrets []*corev1.Secret, needToUpdateSecrets []*corev1.Secret, resultErr error) { @@ -307,27 +323,26 @@ func injectAuthSettings(secret *corev1.Secret, vmuser *vmv1beta1.VMUser) bool { return needUpdate } -var crdNameToObject = map[string]objectWithURL{ - "VMAgent": &vmv1beta1.VMAgent{}, - "VMAlert": &vmv1beta1.VMAlert{}, - "VMSingle": &vmv1beta1.VMSingle{}, - "VLogs": &vmv1beta1.VLogs{}, - // keep both variants for backward-compatibility - "VMAlertmanager": &vmv1beta1.VMAlertmanager{}, - "VMAlertManager": &vmv1beta1.VMAlertmanager{}, - "VMCluster/vmselect": newClusterWithURL("vmselect"), - "VMCluster/vminsert": newClusterWithURL("vminsert"), - "VMCluster/vmstorage": newClusterWithURL("vmstorage"), - "VMAnomaly": &vmv1.VMAnomaly{}, - "VLSingle": &vmv1.VLSingle{}, - "VLCluster/vlselect": newClusterWithURL("vlselect"), - "VLCluster/vlinsert": newClusterWithURL("vlinsert"), - "VLCluster/vlstorage": newClusterWithURL("vlstorage"), - "VLAgent": &vmv1.VLAgent{}, - "VTSingle": &vmv1.VTSingle{}, - "VTCluster/vtselect": newClusterWithURL("vtselect"), - "VTCluster/vtinsert": newClusterWithURL("vtinsert"), - "VTCluster/vtstorage": newClusterWithURL("vtstorage"), +var crdNameToObject = map[string]func() objectWithURL{ + "VMAgent": func() objectWithURL { return &vmv1beta1.VMAgent{} }, + "VMAlert": func() objectWithURL { return &vmv1beta1.VMAlert{} }, + "VMSingle": func() objectWithURL { return &vmv1beta1.VMSingle{} }, + "VLogs": func() objectWithURL { return &vmv1beta1.VLogs{} }, + "VMAlertmanager": func() objectWithURL { return &vmv1beta1.VMAlertmanager{} }, + "VMAlertManager": func() objectWithURL { return &vmv1beta1.VMAlertmanager{} }, + "VMCluster/vmselect": func() objectWithURL { return newClusterWithURL("vmselect") }, + "VMCluster/vminsert": func() objectWithURL { return newClusterWithURL("vminsert") }, + "VMCluster/vmstorage": func() objectWithURL { return newClusterWithURL("vmstorage") }, + "VMAnomaly": func() objectWithURL { return &vmv1.VMAnomaly{} }, + "VLSingle": func() objectWithURL { return &vmv1.VLSingle{} }, + "VLCluster/vlselect": func() objectWithURL { return newClusterWithURL("vlselect") }, + "VLCluster/vlinsert": func() objectWithURL { return newClusterWithURL("vlinsert") }, + "VLCluster/vlstorage": func() objectWithURL { return newClusterWithURL("vlstorage") }, + "VLAgent": func() objectWithURL { return &vmv1.VLAgent{} }, + "VTSingle": func() objectWithURL { return &vmv1.VTSingle{} }, + "VTCluster/vtselect": func() objectWithURL { return newClusterWithURL("vtselect") }, + "VTCluster/vtinsert": func() objectWithURL { return newClusterWithURL("vtinsert") }, + "VTCluster/vtstorage": func() objectWithURL { return newClusterWithURL("vtstorage") }, } // helper interface to restore VMCluster type @@ -335,40 +350,53 @@ type unwrapObject interface { origin() client.Object } -var clusterComponentToURL = map[string]func(obj client.Object, isExtra bool) string{ - "vminsert": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) +// rejectPool wraps a pool-unaware cluster AsURL so its kind rejects a non-empty pool instead of +// silently ignoring it - only VMCluster currently supports spec.pools. +func rejectPool(kind string, f func(obj client.Object, isExtra bool) (string, error)) func(obj client.Object, poolName string, isExtra bool) (string, error) { + return func(obj client.Object, poolName string, isExtra bool) (string, error) { + if poolName != "" { + return "", fmt.Errorf("pool=%q is not supported for kind=%q", poolName, kind) + } + return f(obj, isExtra) + } +} + +var clusterComponentToURL = map[string]func(obj client.Object, poolName string, isExtra bool) (string, error){ + "vminsert": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURLForPool(vmv1beta1.ClusterComponentInsert, poolName, isExtra) }, - "vmselect": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) + "vmselect": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURLForPool(vmv1beta1.ClusterComponentSelect, poolName, isExtra) }, - "vmstorage": func(obj client.Object, isExtra bool) string { - return obj.(*vmv1beta1.VMCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) + "vmstorage": func(obj client.Object, poolName string, isExtra bool) (string, error) { + return obj.(*vmv1beta1.VMCluster).AsURLForPool(vmv1beta1.ClusterComponentStorage, poolName, isExtra) }, - "vlinsert": func(obj client.Object, isExtra bool) string { + "vlinsert": rejectPool("VLCluster/vlinsert", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) - }, - "vlselect": func(obj client.Object, isExtra bool) string { + }), + "vlselect": rejectPool("VLCluster/vlselect", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) - }, - "vlstorage": func(obj client.Object, isExtra bool) string { + }), + "vlstorage": rejectPool("VLCluster/vlstorage", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VLCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) - }, - "vtinsert": func(obj client.Object, isExtra bool) string { + }), + "vtinsert": rejectPool("VTCluster/vtinsert", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentInsert, isExtra) - }, - "vtselect": func(obj client.Object, isExtra bool) string { + }), + "vtselect": rejectPool("VTCluster/vtselect", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentSelect, isExtra) - }, - "vtstorage": func(obj client.Object, isExtra bool) string { + }), + "vtstorage": rejectPool("VTCluster/vtstorage", func(obj client.Object, isExtra bool) (string, error) { return obj.(*vmv1.VTCluster).AsURL(vmv1beta1.ClusterComponentStorage, isExtra) - }, + }), } type clusterWithURL struct { client.Object originObj client.Object component string + pool string + err error } func newClusterWithURL(component string) *clusterWithURL { @@ -383,20 +411,25 @@ func newClusterWithURL(component string) *clusterWithURL { default: panic(fmt.Sprintf("BUG: unexpected component name: %q", component)) } - return &clusterWithURL{clusterObj, clusterObj, component} + return &clusterWithURL{Object: clusterObj, originObj: clusterObj, component: component} } func (c *clusterWithURL) origin() client.Object { return c.originObj } -// AsURL implements AsURL interface func (c *clusterWithURL) AsURL(isExtra bool) string { builder, ok := clusterComponentToURL[c.component] if !ok { panic(fmt.Sprintf("BUG: not expected component=%q for clusterWithURL object", c.component)) } - return builder(c.Object, isExtra) + url, err := builder(c.Object, c.pool, isExtra) + c.err = err + return url +} + +func (c *clusterWithURL) URLError() error { + return c.err } // generateVMAuthConfig create VMAuth cfg for given Users. diff --git a/internal/controller/operator/factory/vmauth/vmusers_config_test.go b/internal/controller/operator/factory/vmauth/vmusers_config_test.go index 17d1099da1..62893cb6e0 100644 --- a/internal/controller/operator/factory/vmauth/vmusers_config_test.go +++ b/internal/controller/operator/factory/vmauth/vmusers_config_test.go @@ -76,8 +76,8 @@ func Test_genUserCfg(t *testing.T) { }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -155,8 +155,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -203,8 +203,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMCluster/vminsert/monitoring/vminsert": "http://vminsert.monitoring.svc:8481", - "VMCluster/vmselect/monitoring/vmselect": "http://vmselect.monitoring.svc:8482", + "VMCluster/vminsert/monitoring/vminsert/": "http://vminsert.monitoring.svc:8481", + "VMCluster/vmselect/monitoring/vmselect/": "http://vmselect.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -287,8 +287,8 @@ password: pass }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_map: - url_prefix: @@ -360,9 +360,9 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", - "VLogs/monitoring/db": "http://vlogs-b.monitoring.svc:8482", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", + "VLogs/monitoring/db/": "http://vlogs-b.monitoring.svc:8482", }, want: `url_map: - url_prefix: @@ -409,8 +409,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_prefix: - http://vmagent-base.monitoring.svc:8429 @@ -442,8 +442,8 @@ bearer_token: secret-token }, }, objURLs: map[string]string{ - "VMAgent/monitoring/base": "http://vmagent-base.monitoring.svc:8429", - "VMSingle/monitoring/db": "http://vmsingle-b.monitoring.svc:8429", + "VMAgent/monitoring/base/": "http://vmagent-base.monitoring.svc:8429", + "VMSingle/monitoring/db/": "http://vmsingle-b.monitoring.svc:8429", }, want: `url_prefix: - http://vmagent-base.monitoring.svc:8429 @@ -740,11 +740,11 @@ password: pass }, }, objURLs: map[string]string{ - "VLAgent/monitoring/collector": "http://vlagent-base.monitoring.svc:9429", - "VLSingle/monitoring/db": "http://vlsingle-db.monitoring.svc:9428", - "VLCluster/vlinsert/monitoring/main-cluster": "http://vlinsert-main-cluster.monitoring.svc:9401", - "VLCluster/vlselect/monitoring/main-cluster": "http://vlselect-main-cluster.monitoring.svc:9401", - "VLCluster/vlstorage/monitoring/main-cluster": "http://vlstorage-main-cluster.monitoring.svc:9401", + "VLAgent/monitoring/collector/": "http://vlagent-base.monitoring.svc:9429", + "VLSingle/monitoring/db/": "http://vlsingle-db.monitoring.svc:9428", + "VLCluster/vlinsert/monitoring/main-cluster/": "http://vlinsert-main-cluster.monitoring.svc:9401", + "VLCluster/vlselect/monitoring/main-cluster/": "http://vlselect-main-cluster.monitoring.svc:9401", + "VLCluster/vlstorage/monitoring/main-cluster/": "http://vlstorage-main-cluster.monitoring.svc:9401", }, want: `url_map: - url_prefix: @@ -1198,8 +1198,10 @@ func Test_buildConfig(t *testing.T) { Namespace: "monitoring", }, Spec: vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{}, - VMInsert: &vmv1beta1.VMInsert{}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{}, + VMInsert: &vmv1beta1.VMInsert{}, + }, }, }, &vmv1beta1.VMAgent{ @@ -2469,14 +2471,16 @@ unauthorized_user: Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(10)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(10)), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(5)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(5)), + }, }, }, }, @@ -3310,17 +3314,19 @@ unauthorized_user: &vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - {Name: "http", Port: 18480}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + {Name: "http", Port: 18480}, + }, }, }, }, + VMSelect: &vmv1beta1.VMSelect{}, + VMStorage: &vmv1beta1.VMStorage{}, }, - VMSelect: &vmv1beta1.VMSelect{}, - VMStorage: &vmv1beta1.VMStorage{}, }, }, &vmv1beta1.VMUser{ @@ -3350,6 +3356,56 @@ unauthorized_user: src_paths: - /insert/.* bearer_token: bearer +`, + }) + + // VMUser targeting a specific pool's vminsert on a VMCluster where every pool has its own + // dedicated insert (the shared top-level vminsert is ambiguous in that case). + f(opts{ + cr: &vmv1beta1.VMAuth{ + ObjectMeta: metav1.ObjectMeta{Name: "test-auth", Namespace: "default"}, + Spec: vmv1beta1.VMAuthSpec{SelectAllByDefault: true}, + }, + predefinedObjects: []runtime.Object{ + &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "pooled", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{}, + }, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMInsert: &vmv1beta1.VMInsert{}}, + {Name: "cold", VMInsert: &vmv1beta1.VMInsert{}}, + }, + }, + }, + &vmv1beta1.VMUser{ + ObjectMeta: metav1.ObjectMeta{Name: "user-1", Namespace: "default"}, + Spec: vmv1beta1.VMUserSpec{ + BearerToken: ptr.To("bearer"), + TargetRefs: []vmv1beta1.TargetRef{ + { + CRD: &vmv1beta1.CRDRef{ + Kind: "VMCluster/vminsert", + NamespacedName: vmv1beta1.NamespacedName{ + Name: "pooled", + Namespace: "default", + Pool: "hot", + }, + }, + Paths: []string{"/insert/.*"}, + }, + }, + }, + }, + }, + want: `users: +- url_map: + - url_prefix: + - http://vminsert-pooled-hot.default.svc:8480 + src_paths: + - /insert/.* + bearer_token: bearer `, }) } diff --git a/internal/controller/operator/factory/vmcluster/vmcluster.go b/internal/controller/operator/factory/vmcluster/vmcluster.go index 9c449b8834..c8187b700b 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" vpav1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,13 +29,9 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/reconcile" ) -// CreateOrUpdate reconciled cluster object with order -// first we check status of vmStorage and waiting for its readiness -// then vmSelect and wait for it readiness as well -// and last one is vmInsert -// we manually handle statefulsets rolling updates -// needed in update checked by revision status -// its controlled by k8s controller-manager +// CreateOrUpdate reconciles the cluster in order: vmstorage/vminsert per pool (waiting for +// readiness before moving on), then vmselect. StatefulSet rolling updates are handled manually, +// gated on revision status, since k8s controller-manager doesn't manage that for us here. func CreateOrUpdate(ctx context.Context, cr *vmv1beta1.VMCluster, rclient client.Client) error { if cr.Paused() { return nil @@ -55,6 +52,14 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1beta1.VMCluster, rclient client if cr.Spec.VMInsert != nil && cr.Spec.VMInsert.VPA != nil { return fmt.Errorf("spec.vminsert.vpa is set but VM_VPA_API_ENABLED=true env var was not provided") } + for _, pool := range cr.Spec.Pools { + if pool.VMStorage != nil && pool.VMStorage.VPA != nil { + return fmt.Errorf("spec.pools[%s].vmstorage.vpa is set but VM_VPA_API_ENABLED=true env var was not provided", pool.Name) + } + if pool.VMInsert != nil && pool.VMInsert.VPA != nil { + return fmt.Errorf("spec.pools[%s].vminsert.vpa is set but VM_VPA_API_ENABLED=true env var was not provided", pool.Name) + } + } } var prevCR *vmv1beta1.VMCluster if cr.Status.LastAppliedSpec != nil { @@ -82,85 +87,38 @@ func CreateOrUpdate(ctx context.Context, cr *vmv1beta1.VMCluster, rclient client } } - if cr.Spec.VMStorage != nil { - if cr.Spec.VMStorage.PodDisruptionBudget != nil { - err := createOrUpdatePodDisruptionBudgetForVMStorage(ctx, rclient, cr, prevCR) - if err != nil { - return err - } - } - if cr.Spec.VMStorage.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMStorage(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if err := createOrUpdateVMStorage(ctx, rclient, cr, prevCR); err != nil { - return err - } + // base (non-pool) resources; createOrUpdatePool itself decides whether the shared + // top-level vmstorage/vminsert should still be created once pools are declared. + if err := createOrUpdatePool(ctx, rclient, cr, prevCR, nil, owner); err != nil { + return err + } - if err := createOrUpdateVMStorageService(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMStorageHPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMStorageVPA(ctx, rclient, cr, prevCR); err != nil { - return err + for _, pool := range cr.Spec.Pools { + if err := createOrUpdatePool(ctx, rclient, cr, prevCR, &pool, owner); err != nil { + return fmt.Errorf("pool %q: %w", pool.Name, err) } } if cr.Spec.VMSelect != nil { - if cr.Spec.VMSelect.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMSelect(ctx, rclient, cr, prevCR); err != nil { - return err - } + if err := createOrUpdatePodDisruptionBudgetForVMSelect(ctx, rclient, cr, prevCR); err != nil { + return err } - if cr.Spec.VMSelect.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMSelect(ctx, rclient, cr, prevCR); err != nil { - return err - } + if err := createOrUpdateNetworkPolicyForVMSelect(ctx, rclient, cr, prevCR); err != nil { + return err } if err := createOrUpdateVMSelect(ctx, rclient, cr, prevCR); err != nil { return err } - if err := createOrUpdateVMSelectHPA(ctx, rclient, cr, prevCR); err != nil { return err } if err := createOrUpdateVMSelectVPA(ctx, rclient, cr, prevCR); err != nil { return err } - // create vmselect service if err := createOrUpdateVMSelectService(ctx, rclient, cr, prevCR); err != nil { return err } } - - if cr.Spec.VMInsert != nil { - if cr.Spec.VMInsert.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if cr.Spec.VMInsert.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if err := createOrUpdateVMInsert(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertService(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertHPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - if err := createOrUpdateVMInsertVPA(ctx, rclient, cr, prevCR); err != nil { - return err - } - } - if prevCR != nil { if err := deleteOrphaned(ctx, rclient, cr); err != nil { return fmt.Errorf("failed to remove objects from previous cluster state: %w", err) @@ -321,21 +279,22 @@ func createOrUpdateLBProxyService(ctx context.Context, rclient client.Client, cr return nil } -func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string, owner metav1.OwnerReference) error { var prevDeploy *appsv1.Deployment if prevCR != nil && prevCR.Spec.VMInsert != nil { var err error - prevDeploy, err = genVMInsertSpec(prevCR) + prevDeploy, err = genVMInsertSpec(prevCR, poolName) if err != nil { return fmt.Errorf("cannot generate prev deploy spec: %w", err) } + prevDeploy.OwnerReferences = []metav1.OwnerReference{owner} } - newDeployment, err := genVMInsertSpec(cr) + newDeployment, err := genVMInsertSpec(cr, poolName) if err != nil { return err } - owner := cr.AsOwner() + newDeployment.OwnerReferences = []metav1.OwnerReference{owner} o := reconcile.DeploymentOpts{ PatchSpec: func(existingSpec, newSpec *appsv1.DeploymentSpec) { if cr.Spec.VMInsert.HPA != nil { @@ -347,8 +306,8 @@ func createOrUpdateVMInsert(ctx context.Context, rclient client.Client, cr, prev return reconcile.Deployment(ctx, rclient, newDeployment, prevDeploy, &owner, &o) } -func buildVMInsertService(cr *vmv1beta1.VMCluster) *corev1.Service { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func buildVMInsertService(cr *vmv1beta1.VMCluster, poolName string) *corev1.Service { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) svc := build.Service(b, cr.Spec.VMInsert.Port, func(svc *corev1.Service) { build.AppendInsertPortsToService(cr.Spec.VMInsert.InsertPorts, svc) if cr.Spec.VMInsert.ClusterNativePort != "" { @@ -381,19 +340,20 @@ func buildVMInsertScrape(cr *vmv1beta1.VMCluster, svc *corev1.Service) *vmv1beta return svs } -func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - svc := buildVMInsertService(cr) +func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, owner metav1.OwnerReference, poolName string) error { + svc := buildVMInsertService(cr, poolName) + svc.OwnerReferences = []metav1.OwnerReference{owner} var prevSvc, prevAdditionalSvc *corev1.Service if prevCR != nil && prevCR.Spec.VMInsert != nil { - prevSvc = buildVMInsertService(prevCR) + prevSvc = buildVMInsertService(prevCR, poolName) + prevSvc.OwnerReferences = []metav1.OwnerReference{owner} prevAdditionalSvcBase := *prevSvc - prevAdditionalSvcBase.Name = prevCR.PrefixedName(vmv1beta1.ClusterComponentInsert) + prevAdditionalSvcBase.Name = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName).PrefixedName() prevAdditionalSvc = build.AdditionalServiceFromDefault(&prevAdditionalSvcBase, prevCR.Spec.VMInsert.ServiceSpec) } - owner := cr.AsOwner() if err := cr.Spec.VMInsert.ServiceSpec.IsSomeAndThen(func(s *vmv1beta1.AdditionalServiceSpec) error { additionalSvcBase := *svc - additionalSvcBase.Name = cr.PrefixedName(vmv1beta1.ClusterComponentInsert) + additionalSvcBase.Name = build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName).PrefixedName() additionalSvc := build.AdditionalServiceFromDefault(&additionalSvcBase, s) if additionalSvc.Name == svc.Name { return fmt.Errorf("vminsert additional service name: %q cannot be the same as crd.prefixedname: %q", additionalSvc.Name, svc.Name) @@ -431,23 +391,25 @@ func createOrUpdateVMInsertService(ctx context.Context, rclient client.Client, c return nil } -func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string, owner metav1.OwnerReference) error { var prevSts *appsv1.StatefulSet if prevCR != nil && prevCR.Spec.VMStorage != nil { var err error - prevSts, err = buildVMStorageSpec(ctx, prevCR) + prevSts, err = buildVMStorageSpec(ctx, prevCR, poolName) if err != nil { return fmt.Errorf("cannot build prev storage spec: %w", err) } + prevSts.OwnerReferences = []metav1.OwnerReference{owner} } - newSts, err := buildVMStorageSpec(ctx, cr) + newSts, err := buildVMStorageSpec(ctx, cr, poolName) if err != nil { return err } + newSts.OwnerReferences = []metav1.OwnerReference{owner} o := reconcile.StatefulSetOpts{ - SelectorLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentStorage), + SelectorLabels: newSts.Spec.Selector.MatchLabels, UpdateBehavior: cr.Spec.VMStorage.RollingUpdateStrategyBehavior, PatchSpec: func(existingSpec, newSpec *appsv1.StatefulSetSpec) { if cr.Spec.VMStorage.HPA != nil { @@ -456,12 +418,11 @@ func createOrUpdateVMStorage(ctx context.Context, rclient client.Client, cr, pre } }, } - owner := cr.AsOwner() return reconcile.StatefulSet(ctx, rclient, newSts, prevSts, &owner, &o) } -func buildVMStorageService(cr *vmv1beta1.VMCluster) *corev1.Service { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func buildVMStorageService(cr *vmv1beta1.VMCluster, poolName string) *corev1.Service { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) return build.Service(b, cr.Spec.VMStorage.Port, func(svc *corev1.Service) { svc.Spec.ClusterIP = "None" svc.Spec.PublishNotReadyAddresses = true @@ -498,14 +459,15 @@ func buildVMStorageScrape(cr *vmv1beta1.VMCluster, svc *corev1.Service) *vmv1bet return build.VMServiceScrape(svc, cr.Spec.VMStorage, "vmbackupmanager") } -func createOrUpdateVMStorageService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - svc := buildVMStorageService(cr) +func createOrUpdateVMStorageService(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, owner metav1.OwnerReference, poolName string) error { + svc := buildVMStorageService(cr, poolName) + svc.OwnerReferences = []metav1.OwnerReference{owner} var prevSvc, prevAdditionalSvc *corev1.Service if prevCR != nil && prevCR.Spec.VMStorage != nil { - prevSvc = buildVMStorageService(prevCR) + prevSvc = buildVMStorageService(prevCR, poolName) + prevSvc.OwnerReferences = []metav1.OwnerReference{owner} prevAdditionalSvc = build.AdditionalServiceFromDefault(prevSvc, prevCR.Spec.VMStorage.ServiceSpec) } - owner := cr.AsOwner() if err := cr.Spec.VMStorage.ServiceSpec.IsSomeAndThen(func(s *vmv1beta1.AdditionalServiceSpec) error { additionalSvc := build.AdditionalServiceFromDefault(svc, s) if additionalSvc.Name == svc.Name { @@ -607,6 +569,17 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e args = append(args, fmt.Sprintf("-replicationFactor=%d", *cr.Spec.ReplicationFactor)) } } + if cr.Spec.GlobalReplicationFactor != nil { + var globalReplicationFactorIsSet bool + for arg := range cr.Spec.VMSelect.ExtraArgs { + if strings.Contains(arg, "globalReplicationFactor") { + globalReplicationFactorIsSet = true + } + } + if !globalReplicationFactorIsSet { + args = append(args, fmt.Sprintf("-globalReplicationFactor=%d", *cr.Spec.GlobalReplicationFactor)) + } + } if cr.Spec.Downsampling.HasAnyRule() { for _, rule := range cr.Spec.Downsampling.Rules { for _, p := range rule.Periods { @@ -623,30 +596,62 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } storageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if d := cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { - storageNode := fmt.Sprintf("srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)) - for _, node := range cr.Spec.VMSelect.ExtraStorageNodes { - storageNode += "," + node.Addr + storageNodeFlag := build.NewFlag("-storageNode", "") + var baseCount int + d := cr.Spec.VMSelect.Discovery.OrDefault(cr.Spec.Discovery) + if len(cr.Spec.Pools) == 0 { + if d != nil && d.Enabled { + storageNodeFlag.Add(fmt.Sprintf("srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName)), 0) + baseCount = 1 + } else { + storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentSelect) + for idx, i := range storageNodeIds { + storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName), idx) + } + baseCount = len(storageNodeIds) } - args = append(args, fmt.Sprintf("-storageNode=%s", storageNode)) + } + + // Pool storage nodes — each pool is exposed as a named storage group. + poolCount := baseCount + for _, pool := range cr.Spec.Pools { + poolStr, err := poolStorage(cr, &pool) + if err != nil { + return nil, fmt.Errorf("pool %q: cannot build storage spec: %w", pool.Name, err) + } + if poolStr == nil { + continue + } + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + if d != nil && d.Enabled { + storageNodeFlag.Add(fmt.Sprintf("%s/srv+%s", pool.Name, storageNodeSRVAddr(poolStorageName, cr.Namespace, poolStr.VMSelectPort, cr.Spec.ClusterDomainName)), poolCount) + poolCount++ + } else { + for _, i := range vmv1beta1.AvailableStorageNodeIDsFor(poolStr, vmv1beta1.ClusterComponentSelect) { + addr := vmv1beta1.PodDNSAddress(poolStorageName, i, cr.Namespace, poolStr.VMSelectPort, cr.Spec.ClusterDomainName) + storageNodeFlag.Add(fmt.Sprintf("%s/%s", pool.Name, addr), poolCount) + poolCount++ + } + } + if pool.ReplicationFactor != nil { + args = append(args, fmt.Sprintf("-replicationFactor=%s:%d", pool.Name, *pool.ReplicationFactor)) + } + } + + if d != nil && d.Enabled { if d.Interval != "" { args = append(args, fmt.Sprintf("-storageNode.discoveryInterval=%s", d.Interval)) } if d.Filter != "" { args = append(args, fmt.Sprintf("-storageNode.filter=%s", d.Filter)) } - } else { - storageNodeFlag := build.NewFlag("-storageNode", "") - storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentSelect) - for idx, i := range storageNodeIds { - storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMSelectPort, cr.Spec.ClusterDomainName), idx) - } - for i, node := range cr.Spec.VMSelect.ExtraStorageNodes { - storageNodeFlag.Add(node.Addr, i+len(storageNodeIds)) - } - args = build.AppendFlagsToArgs(args, len(storageNodeIds)+len(cr.Spec.VMSelect.ExtraStorageNodes), storageNodeFlag) } + for i, node := range cr.Spec.VMSelect.ExtraStorageNodes { + storageNodeFlag.Add(node.Addr, poolCount+i) + } + args = build.AppendFlagsToArgs(args, poolCount+len(cr.Spec.VMSelect.ExtraStorageNodes), storageNodeFlag) + // selectNode arg add for deployments without HPA // HPA leads to rolling restart for vmselect statefulset in case of replicas count changes if cr.Spec.VMSelect.HPA == nil && cr.Spec.VMSelect.ReplicaCount != nil { @@ -775,6 +780,9 @@ func makePodSpecForVMSelect(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } func createOrUpdatePodDisruptionBudgetForVMSelect(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.VMSelect.PodDisruptionBudget == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentSelect) pdb := build.PodDisruptionBudget(b, cr.Spec.VMSelect.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget @@ -787,6 +795,9 @@ func createOrUpdatePodDisruptionBudgetForVMSelect(ctx context.Context, rclient c } func createOrUpdateNetworkPolicyForVMSelect(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.VMSelect.NetworkPolicy == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentSelect) np := build.NetworkPolicy(b, cr.Spec.VMSelect.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy @@ -798,23 +809,26 @@ func createOrUpdateNetworkPolicyForVMSelect(ctx context.Context, rclient client. return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { - - podSpec, err := makePodSpecForVMInsert(cr) +func genVMInsertSpec(cr *vmv1beta1.VMCluster, poolName string) (*appsv1.Deployment, error) { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) + podSpec, err := makePodSpecForVMInsert(cr, poolName) if err != nil { return nil, err } + for k, v := range b.SelectorLabels() { + podSpec.Labels[k] = v + } strategyType := appsv1.RollingUpdateDeploymentStrategyType if cr.Spec.VMInsert.UpdateStrategy != nil { strategyType = *cr.Spec.VMInsert.UpdateStrategy } - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentInsert) + commonName := b.PrefixedName() stsSpec := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: commonName, Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentInsert), + Labels: b.FinalLabels(), Annotations: cr.FinalAnnotations(), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, @@ -827,7 +841,7 @@ func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { RollingUpdate: cr.Spec.VMInsert.RollingUpdate, }, Selector: &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentInsert), + MatchLabels: b.SelectorLabels(), }, Template: *podSpec, }, @@ -836,7 +850,7 @@ func genVMInsertSpec(cr *vmv1beta1.VMCluster) (*appsv1.Deployment, error) { return stsSpec, nil } -func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, error) { +func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster, poolName string) (*corev1.PodTemplateSpec, error) { cfg := config.MustGetBaseConfig() args := []string{ fmt.Sprintf("-httpListenAddr=:%s", cr.Spec.VMInsert.Port), @@ -857,21 +871,53 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e } storageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if d := cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery); d != nil && d.Enabled { - args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName))) + if poolName != "" { + storageName = cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName) + } + d := cr.Spec.VMInsert.Discovery.OrDefault(cr.Spec.Discovery) + if len(cr.Spec.Pools) == 0 { + if d != nil && d.Enabled { + args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(storageName, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName))) + } else { + storageNodeFlag := build.NewFlag("-storageNode", "") + storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentInsert) + for idx, i := range storageNodeIds { + storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName), idx) + } + args = build.AppendFlagsToArgs(args, len(storageNodeIds), storageNodeFlag) + } + } + + // Pool storage nodes for pools using the shared vminsert (no dedicated insert). + for _, pool := range cr.Spec.Pools { + if pool.VMInsert != nil { + continue + } + poolStr, err := poolStorage(cr, &pool) + if err != nil { + return nil, fmt.Errorf("pool %q: cannot build storage spec: %w", pool.Name, err) + } + if poolStr == nil { + continue + } + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + if d != nil && d.Enabled { + args = append(args, fmt.Sprintf("-storageNode=srv+%s", storageNodeSRVAddr(poolStorageName, cr.Namespace, poolStr.VMInsertPort, cr.Spec.ClusterDomainName))) + } else { + for _, i := range vmv1beta1.AvailableStorageNodeIDsFor(poolStr, vmv1beta1.ClusterComponentInsert) { + addr := vmv1beta1.PodDNSAddress(poolStorageName, i, cr.Namespace, poolStr.VMInsertPort, cr.Spec.ClusterDomainName) + args = append(args, fmt.Sprintf("-storageNode=%s", addr)) + } + } + } + + if d != nil && d.Enabled { if d.Interval != "" { args = append(args, fmt.Sprintf("-storageNode.discoveryInterval=%s", d.Interval)) } if d.Filter != "" { args = append(args, fmt.Sprintf("-storageNode.filter=%s", d.Filter)) } - } else { - storageNodeFlag := build.NewFlag("-storageNode", "") - storageNodeIds := cr.AvailableStorageNodeIDs(vmv1beta1.ClusterComponentInsert) - for idx, i := range storageNodeIds { - storageNodeFlag.Add(vmv1beta1.PodDNSAddress(storageName, i, cr.Namespace, cr.Spec.VMStorage.VMInsertPort, cr.Spec.ClusterDomainName), idx) - } - args = build.AppendFlagsToArgs(args, len(storageNodeIds), storageNodeFlag) } if cr.Spec.ReplicationFactor != nil { @@ -976,7 +1022,7 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e for i := range cr.Spec.VMInsert.TopologySpreadConstraints { if cr.Spec.VMInsert.TopologySpreadConstraints[i].LabelSelector == nil { cr.Spec.VMInsert.TopologySpreadConstraints[i].LabelSelector = &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentInsert), + MatchLabels: build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName).SelectorLabels(), } } } @@ -997,49 +1043,60 @@ func makePodSpecForVMInsert(cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, e return vmInsertPodSpec, nil } -func createOrUpdatePodDisruptionBudgetForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func createOrUpdatePodDisruptionBudgetForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMInsert.PodDisruptionBudget == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) pdb := build.PodDisruptionBudget(b, cr.Spec.VMInsert.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget - if prevCR != nil && prevCR.Spec.VMInsert.PodDisruptionBudget != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.PodDisruptionBudget != nil { + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevPDB = build.PodDisruptionBudget(b, prevCR.Spec.VMInsert.PodDisruptionBudget) } owner := cr.AsOwner() return reconcile.PDB(ctx, rclient, pdb, prevPDB, &owner) } -func createOrUpdateNetworkPolicyForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) +func createOrUpdateNetworkPolicyForVMInsert(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMInsert.NetworkPolicy == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) np := build.NetworkPolicy(b, cr.Spec.VMInsert.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.NetworkPolicy != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevNP = build.NetworkPolicy(b, prevCR.Spec.VMInsert.NetworkPolicy) } owner := cr.AsOwner() return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func buildVMStorageSpec(ctx context.Context, cr *vmv1beta1.VMCluster) (*appsv1.StatefulSet, error) { - - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - podSpec, err := makePodSpecForVMStorage(ctx, cr) +func buildVMStorageSpec(ctx context.Context, cr *vmv1beta1.VMCluster, poolName string) (*appsv1.StatefulSet, error) { + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) + commonName := b.PrefixedName() + podSpec, err := makePodSpecForVMStorage(ctx, cr, poolName) if err != nil { return nil, err } + // Merge selector labels into pod template so the STS selector matches its pods. + // For pools this adds the pool label required to keep per-pool selectors disjoint. + for k, v := range b.SelectorLabels() { + podSpec.Labels[k] = v + } stsSpec := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: commonName, Namespace: cr.Namespace, - Labels: cr.FinalLabels(vmv1beta1.ClusterComponentStorage), + Labels: b.FinalLabels(), Annotations: cr.FinalAnnotations(), OwnerReferences: []metav1.OwnerReference{cr.AsOwner()}, }, Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentStorage), + MatchLabels: b.SelectorLabels(), }, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: cr.Spec.VMStorage.RollingUpdateStrategy, @@ -1061,7 +1118,7 @@ func buildVMStorageSpec(ctx context.Context, cr *vmv1beta1.VMCluster) (*appsv1.S return stsSpec, nil } -func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*corev1.PodTemplateSpec, error) { +func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster, poolName string) (*corev1.PodTemplateSpec, error) { cfg := config.MustGetBaseConfig() args := []string{ fmt.Sprintf("-vminsertAddr=:%s", cr.Spec.VMStorage.VMInsertPort), @@ -1071,7 +1128,9 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor if cfg.EnableTCP6 { args = append(args, "-enableTCP6") } - if cr.Spec.RetentionPeriod != "" { + if rp := cr.Spec.VMStorage.RetentionPeriod; rp != "" { + args = append(args, fmt.Sprintf("-retentionPeriod=%s", rp)) + } else if cr.Spec.RetentionPeriod != "" { args = append(args, fmt.Sprintf("-retentionPeriod=%s", cr.Spec.RetentionPeriod)) } @@ -1258,7 +1317,7 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor for i := range cr.Spec.VMStorage.TopologySpreadConstraints { if cr.Spec.VMStorage.TopologySpreadConstraints[i].LabelSelector == nil { cr.Spec.VMStorage.TopologySpreadConstraints[i].LabelSelector = &metav1.LabelSelector{ - MatchLabels: cr.SelectorLabels(vmv1beta1.ClusterComponentStorage), + MatchLabels: build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName).SelectorLabels(), } } } @@ -1279,35 +1338,41 @@ func makePodSpecForVMStorage(ctx context.Context, cr *vmv1beta1.VMCluster) (*cor return vmStoragePodSpec, nil } -func createOrUpdatePodDisruptionBudgetForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func createOrUpdatePodDisruptionBudgetForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMStorage.PodDisruptionBudget == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) pdb := build.PodDisruptionBudget(b, cr.Spec.VMStorage.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget - if prevCR != nil && prevCR.Spec.VMStorage.PodDisruptionBudget != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.PodDisruptionBudget != nil { + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevPDB = build.PodDisruptionBudget(b, prevCR.Spec.VMStorage.PodDisruptionBudget) } owner := cr.AsOwner() return reconcile.PDB(ctx, rclient, pdb, prevPDB, &owner) } -func createOrUpdateNetworkPolicyForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) +func createOrUpdateNetworkPolicyForVMStorage(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { + if cr.Spec.VMStorage.NetworkPolicy == nil { + return nil + } + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) np := build.NetworkPolicy(b, cr.Spec.VMStorage.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.NetworkPolicy != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevNP = build.NetworkPolicy(b, prevCR.Spec.VMStorage.NetworkPolicy) } owner := cr.AsOwner() return reconcile.NetworkPolicy(ctx, rclient, np, prevNP, &owner) } -func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { if cr.Spec.VMInsert.HPA == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) targetRef := autoscalingv2.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "Deployment", @@ -1315,8 +1380,8 @@ func createOrUpdateVMInsertHPA(ctx context.Context, rclient client.Client, cr, p } newHPA := build.HPA(b, targetRef, cr.Spec.VMInsert.HPA) var prevHPA *autoscalingv2.HorizontalPodAutoscaler - if prevCR != nil && prevCR.Spec.VMInsert.HPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.HPA != nil { + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevHPA = build.HPA(b, targetRef, prevCR.Spec.VMInsert.HPA) } owner := cr.AsOwner() @@ -1343,12 +1408,12 @@ func createOrUpdateVMSelectHPA(ctx context.Context, rclient client.Client, cr, p return reconcile.HPA(ctx, rclient, defaultHPA, prevHPA, &owner) } -func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { hpa := cr.Spec.VMStorage.HPA if hpa == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) targetRef := autoscalingv2.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "StatefulSet", @@ -1356,19 +1421,19 @@ func createOrUpdateVMStorageHPA(ctx context.Context, rclient client.Client, cr, } defaultHPA := build.HPA(b, targetRef, hpa) var prevHPA *autoscalingv2.HorizontalPodAutoscaler - if prevCR != nil && prevCR.Spec.VMStorage.HPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.HPA != nil { + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevHPA = build.HPA(b, targetRef, prevCR.Spec.VMStorage.HPA) } owner := cr.AsOwner() return reconcile.HPA(ctx, rclient, defaultHPA, prevHPA, &owner) } -func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { if cr.Spec.VMInsert.VPA == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentInsert, poolName) targetRef := autoscalingv1.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "Deployment", @@ -1377,7 +1442,7 @@ func createOrUpdateVMInsertVPA(ctx context.Context, rclient client.Client, cr, p newVPA := build.VPA(b, targetRef, cr.Spec.VMInsert.VPA) var prevVPA *vpav1.VerticalPodAutoscaler if prevCR != nil && prevCR.Spec.VMInsert != nil && prevCR.Spec.VMInsert.VPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentInsert) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentInsert, poolName) prevVPA = build.VPA(b, targetRef, prevCR.Spec.VMInsert.VPA) } owner := cr.AsOwner() @@ -1404,12 +1469,12 @@ func createOrUpdateVMSelectVPA(ctx context.Context, rclient client.Client, cr, p return reconcile.VPA(ctx, rclient, newVPA, prevVPA, &owner) } -func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { +func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, poolName string) error { vpa := cr.Spec.VMStorage.VPA if vpa == nil { return nil } - b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + b := build.NewPoolBuilder(cr, vmv1beta1.ClusterComponentStorage, poolName) targetRef := autoscalingv1.CrossVersionObjectReference{ Name: b.PrefixedName(), Kind: "StatefulSet", @@ -1418,7 +1483,7 @@ func createOrUpdateVMStorageVPA(ctx context.Context, rclient client.Client, cr, newVPA := build.VPA(b, targetRef, vpa) var prevVPA *vpav1.VerticalPodAutoscaler if prevCR != nil && prevCR.Spec.VMStorage != nil && prevCR.Spec.VMStorage.VPA != nil { - b = build.NewChildBuilder(prevCR, vmv1beta1.ClusterComponentStorage) + b = build.NewPoolBuilder(prevCR, vmv1beta1.ClusterComponentStorage, poolName) prevVPA = build.VPA(b, targetRef, prevCR.Spec.VMStorage.VPA) } owner := cr.AsOwner() @@ -1431,32 +1496,60 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM newInsert := cr.Spec.VMInsert newLB := cr.Spec.RequestsLoadBalancer + hasPools := len(cr.Spec.Pools) > 0 + hasPoolInsert := vmv1beta1.HasAnyPoolInsert(cr.Spec.Pools) + cc := finalize.NewChildCleaner() - if newStorage == nil { - if err := finalize.OnStorageDelete(ctx, rclient, cr, true); err != nil { - return fmt.Errorf("cannot remove orphaned storage resources: %w", err) + + keepStorageResources := func(name string, storage *vmv1beta1.VMStorage) { + cc.KeepService(name) + if storage.ServiceSpec != nil && !storage.ServiceSpec.UseAsDefault { + cc.KeepService(storage.ServiceSpec.NameOrDefault(name)) } - } else { - commonName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) - if newStorage.PodDisruptionBudget != nil { - cc.KeepPDB(commonName) + if !ptr.Deref(storage.DisableSelfServiceScrape, false) { + cc.KeepScrape(name) } - if newStorage.NetworkPolicy != nil { - cc.KeepNetworkPolicy(commonName) + if storage.PodDisruptionBudget != nil { + cc.KeepPDB(name) } - if newStorage.HPA != nil { - cc.KeepHPA(commonName) + if storage.HPA != nil { + cc.KeepHPA(name) } - if newStorage.VPA != nil { - cc.KeepVPA(commonName) + if storage.VPA != nil { + cc.KeepVPA(name) } - if !ptr.Deref(newStorage.DisableSelfServiceScrape, false) { - cc.KeepScrape(commonName) + if storage.NetworkPolicy != nil { + cc.KeepNetworkPolicy(name) } - cc.KeepService(commonName) - if newStorage.ServiceSpec != nil && !newStorage.ServiceSpec.UseAsDefault { - cc.KeepService(newStorage.ServiceSpec.NameOrDefault(commonName)) + } + keepInsertResources := func(name, scrapeName string, insert *vmv1beta1.VMInsert) { + cc.KeepService(name) + if insert.ServiceSpec != nil && !insert.ServiceSpec.UseAsDefault { + cc.KeepService(insert.ServiceSpec.NameOrDefault(name)) + } + if !ptr.Deref(insert.DisableSelfServiceScrape, false) { + cc.KeepScrape(scrapeName) + } + if insert.PodDisruptionBudget != nil { + cc.KeepPDB(name) + } + if insert.HPA != nil { + cc.KeepHPA(name) + } + if insert.VPA != nil { + cc.KeepVPA(name) + } + if insert.NetworkPolicy != nil { + cc.KeepNetworkPolicy(name) + } + } + + if newStorage == nil && !hasPools { + if err := finalize.OnStorageDelete(ctx, rclient, cr, true); err != nil { + return fmt.Errorf("cannot remove orphaned storage resources: %w", err) } + } else if newStorage != nil && !hasPools { + keepStorageResources(cr.PrefixedName(vmv1beta1.ClusterComponentStorage), newStorage) } if newSelect == nil { @@ -1491,36 +1584,18 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM } } - if newInsert == nil { + if newInsert == nil && !hasPoolInsert { if err := finalize.OnInsertDelete(ctx, rclient, cr, true); err != nil { return fmt.Errorf("cannot remove orphaned insert resources: %w", err) } - } else { + } else if newInsert != nil && !hasPoolInsert { commonName := cr.PrefixedName(vmv1beta1.ClusterComponentInsert) - if newInsert.PodDisruptionBudget != nil { - cc.KeepPDB(commonName) - } - if newInsert.NetworkPolicy != nil { - cc.KeepNetworkPolicy(commonName) - } - if newInsert.HPA != nil { - cc.KeepHPA(commonName) - } - if newInsert.VPA != nil { - cc.KeepVPA(commonName) - } - cc.KeepService(commonName) - if newInsert.ServiceSpec != nil && !newInsert.ServiceSpec.UseAsDefault { - cc.KeepService(newInsert.ServiceSpec.NameOrDefault(commonName)) - } scrapeName := commonName if newLB.Enabled && !newLB.DisableInsertBalancing { scrapeName = cr.PrefixedInternalName(vmv1beta1.ClusterComponentInsert) cc.KeepService(scrapeName) } - if !ptr.Deref(newInsert.DisableSelfServiceScrape, false) { - cc.KeepScrape(scrapeName) - } + keepInsertResources(commonName, scrapeName, newInsert) } if newLB.Enabled { commonName := cr.PrefixedName(vmv1beta1.ClusterComponentBalancer) @@ -1553,6 +1628,53 @@ func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VM return fmt.Errorf("cannot remove serviceaccount: %w", err) } } + + // Pool resource cleanup: keep active pool services and remove orphaned pool StatefulSets/Deployments. + keepStorageSTSNames := sets.New[string]() + keepInsertDeploymentNames := sets.New[string]() + if newStorage != nil && !hasPools { + keepStorageSTSNames.Insert(cr.PrefixedName(vmv1beta1.ClusterComponentStorage)) + } + if newInsert != nil && !hasPoolInsert { + keepInsertDeploymentNames.Insert(cr.PrefixedName(vmv1beta1.ClusterComponentInsert)) + } + for _, pool := range cr.Spec.Pools { + poolStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, pool.Name) + poolStorageItem, err := poolStorage(cr, &pool) + if err != nil { + return fmt.Errorf("pool %q: cannot resolve storage: %w", pool.Name, err) + } + if poolStorageItem != nil { + keepStorageSTSNames.Insert(poolStorageName) + keepStorageResources(poolStorageName, poolStorageItem) + } + if pool.VMInsert != nil { + poolInsertName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, pool.Name) + keepInsertDeploymentNames.Insert(poolInsertName) + poolInsertItem, err := poolInsert(cr, pool) + if err != nil { + return fmt.Errorf("pool %q: cannot resolve insert: %w", pool.Name, err) + } + if poolInsertItem != nil { + scrapeName := poolInsertName + if newLB.Enabled && !newLB.DisableInsertBalancing { + scrapeName = cr.PrefixedInternalName(vmv1beta1.ClusterComponentInsert) + cc.KeepService(scrapeName) + cc.KeepService(cr.PrefixedName(vmv1beta1.ClusterComponentInsert)) + } + keepInsertResources(poolInsertName, scrapeName, poolInsertItem) + } + } + } + storageBuilder := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentStorage) + if err := finalize.RemoveOrphanedSTSs(ctx, rclient, storageBuilder, keepStorageSTSNames, true); err != nil { + return fmt.Errorf("cannot remove orphaned pool storage StatefulSets: %w", err) + } + insertBuilder := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentInsert) + if err := finalize.RemoveOrphanedDeployments(ctx, rclient, insertBuilder, keepInsertDeploymentNames, true); err != nil { + return fmt.Errorf("cannot remove orphaned pool insert Deployments: %w", err) + } + return cc.RemoveOrphaned(ctx, rclient, cr) } @@ -1581,9 +1703,18 @@ func buildVMAuthLBSecret(cr *vmv1beta1.VMCluster) *corev1.Secret { selectProto = "https" } } - if cr.Spec.VMInsert != nil { - insertPort = cr.Spec.VMInsert.Port - if cr.Spec.VMInsert.UseTLS() { + insert := cr.Spec.VMInsert + for i := range cr.Spec.Pools { + if cr.Spec.Pools[i].VMInsert != nil { + if resolved, err := cr.ResolvePoolVMInsert(&cr.Spec.Pools[i]); err == nil { + insert = resolved + } + break + } + } + if insert != nil { + insertPort = insert.Port + if insert.UseTLS() { insertProto = "https" } } @@ -1788,15 +1919,11 @@ func createOrUpdateVMAuthLB(ctx context.Context, rclient client.Client, cr, prev if err := createOrUpdateVMAuthLBService(ctx, rclient, cr, prevCR); err != nil { return err } - if cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget != nil { - if err := createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { - return fmt.Errorf("cannot create or update PodDisruptionBudget for vmauth lb: %w", err) - } + if err := createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { + return fmt.Errorf("cannot create or update PodDisruptionBudget for vmauth lb: %w", err) } - if cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy != nil { - if err := createOrUpdateNetworkPolicyForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { - return fmt.Errorf("cannot create or update NetworkPolicy for vmauth lb: %w", err) - } + if err := createOrUpdateNetworkPolicyForVMAuthLB(ctx, rclient, cr, prevCR); err != nil { + return fmt.Errorf("cannot create or update NetworkPolicy for vmauth lb: %w", err) } if err := createOrUpdateVMAuthLBHPA(ctx, rclient, cr, prevCR); err != nil { return fmt.Errorf("cannot create or update HPA for vmauth lb: %w", err) @@ -1834,6 +1961,9 @@ func storageNodeSRVAddr(svcName, namespace, port, clusterDomain string) string { } func createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentBalancer) pdb := build.PodDisruptionBudget(b, cr.Spec.RequestsLoadBalancer.Spec.PodDisruptionBudget) var prevPDB *policyv1.PodDisruptionBudget @@ -1846,6 +1976,9 @@ func createOrUpdatePodDisruptionBudgetForVMAuthLB(ctx context.Context, rclient c } func createOrUpdateNetworkPolicyForVMAuthLB(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster) error { + if cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy == nil { + return nil + } b := build.NewChildBuilder(cr, vmv1beta1.ClusterComponentBalancer) np := build.NetworkPolicy(b, cr.Spec.RequestsLoadBalancer.Spec.NetworkPolicy) var prevNP *networkingv1.NetworkPolicy diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_pools.go b/internal/controller/operator/factory/vmcluster/vmcluster_pools.go new file mode 100644 index 0000000000..33186b8bce --- /dev/null +++ b/internal/controller/operator/factory/vmcluster/vmcluster_pools.go @@ -0,0 +1,145 @@ +package vmcluster + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" +) + +func createOrUpdatePool(ctx context.Context, rclient client.Client, cr, prevCR *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool, owner metav1.OwnerReference) error { + var poolName string + view, err := buildPoolView(cr, pool) + if err != nil { + return err + } + poolVMStorage := cr.Spec.VMStorage + poolVMInsert := cr.Spec.VMInsert + if pool != nil { + poolName = pool.Name + // every declared pool gets its own storage, falling back to the base + // config; insert stays shared unless the pool overrides it. + poolVMStorage = view.Spec.VMStorage + poolVMInsert = pool.VMInsert + } else { + if len(cr.Spec.Pools) > 0 { + poolVMStorage = nil + } + for _, p := range cr.Spec.Pools { + if p.VMInsert != nil { + poolVMInsert = nil + break + } + } + } + var prevView *vmv1beta1.VMCluster + if prevCR != nil { + if pool == nil { + prevView, err = buildPoolView(prevCR, nil) + if err != nil { + return err + } + } else if prevPool, ok := prevCR.FindPool(poolName); ok { + prevView, err = buildPoolView(prevCR, prevPool) + if err != nil { + return err + } + } + } + if poolVMStorage != nil { + if err = createOrUpdatePodDisruptionBudgetForVMStorage(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage pdb: %w", err) + } + if err = createOrUpdateVMStorage(ctx, rclient, view, prevView, poolName, owner); err != nil { + return fmt.Errorf("vmstorage: %w", err) + } + if err = createOrUpdateVMStorageService(ctx, rclient, view, prevView, owner, poolName); err != nil { + return fmt.Errorf("vmstorage service: %w", err) + } + if err = createOrUpdateVMStorageHPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage hpa: %w", err) + } + if err = createOrUpdateVMStorageVPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage vpa: %w", err) + } + if err = createOrUpdateNetworkPolicyForVMStorage(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vmstorage networkPolicy: %w", err) + } + } + if poolVMInsert != nil { + if err = createOrUpdatePodDisruptionBudgetForVMInsert(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert pdb: %w", err) + } + if err = createOrUpdateVMInsert(ctx, rclient, view, prevView, poolName, owner); err != nil { + return fmt.Errorf("vminsert: %w", err) + } + if err = createOrUpdateVMInsertService(ctx, rclient, view, prevView, owner, poolName); err != nil { + return fmt.Errorf("vminsert service: %w", err) + } + if err = createOrUpdateVMInsertHPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert hpa: %w", err) + } + if err = createOrUpdateVMInsertVPA(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert vpa: %w", err) + } + if err = createOrUpdateNetworkPolicyForVMInsert(ctx, rclient, view, prevView, poolName); err != nil { + return fmt.Errorf("vminsert networkPolicy: %w", err) + } + } + return nil +} + +func buildPoolView(cr *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool) (*vmv1beta1.VMCluster, error) { + view := cr.DeepCopy() + storage, err := poolStorage(cr, pool) + if err != nil { + return nil, fmt.Errorf("cannot resolve vmstorage: %w", err) + } + view.Spec.VMStorage = storage + if storage != nil && storage.RetentionPeriod != "" { + view.Spec.RetentionPeriod = storage.RetentionPeriod + } + + if pool != nil { + if pool.VMInsert != nil { + insert, err := poolInsert(cr, *pool) + if err != nil { + return nil, fmt.Errorf("cannot resolve vminsert: %w", err) + } + view.Spec.VMInsert = insert + } else { + view.Spec.VMInsert = nil + } + // a per-pool view must not see sibling pools. + view.Spec.Pools = nil + } + + view.Spec.VMSelect = nil + return view, nil +} + +func poolStorage(cr *vmv1beta1.VMCluster, pool *vmv1beta1.VMClusterPool) (*vmv1beta1.VMStorage, error) { + merged, err := cr.ResolvePoolVMStorage(pool) + if err != nil { + return nil, err + } + if merged != nil && pool != nil && pool.VMStorage != nil && cr.Spec.VMStorage == nil { + build.DefaultPoolVMStorage(merged, cr) + } + return merged, nil +} + +func poolInsert(cr *vmv1beta1.VMCluster, pool vmv1beta1.VMClusterPool) (*vmv1beta1.VMInsert, error) { + merged, err := cr.ResolvePoolVMInsert(&pool) + if err != nil { + return nil, err + } + if merged != nil && cr.Spec.VMInsert == nil { + build.DefaultPoolVMInsert(merged, cr) + } + return merged, nil +} diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go index a56725427e..f6226e471c 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_reconcile_test.go @@ -78,20 +78,22 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { defaultCR := &vmv1beta1.VMCluster{ ObjectMeta: objectMeta, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, }, }, @@ -170,6 +172,16 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, {Verb: "Create", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert - pools reconcile vmstorage and vminsert together per pool, so + // vminsert now runs right after vmstorage, ahead of vmselect. + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Create", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, // wait for ready + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Create", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + {Verb: "Create", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Create", Kind: "StatefulSet", Resource: vmselectName}, @@ -178,15 +190,6 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Create", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, {Verb: "Create", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Create", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, // wait for ready - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Create", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, - {Verb: "Create", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) @@ -206,18 +209,18 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "Service", Resource: vmstorageName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // getLatestStsState {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // patchSTSCurrentRevision {Verb: "Get", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) @@ -242,18 +245,18 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { {Verb: "Get", Kind: "Service", Resource: vmstorageName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmstorageName}, + // VMInsert + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, + {Verb: "Get", Kind: "Service", Resource: vminsertName}, + {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, + // VMSelect {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // getLatestStsState {Verb: "Get", Kind: "StatefulSet", Resource: vmselectName}, // patchSTSCurrentRevision {Verb: "Get", Kind: "Service", Resource: vmselectName}, {Verb: "Get", Kind: "VMServiceScrape", Resource: vmselectName}, - - // VMInsert - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Deployment", Resource: vminsertName}, - {Verb: "Get", Kind: "Service", Resource: vminsertName}, - {Verb: "Get", Kind: "VMServiceScrape", Resource: vminsertName}, }, }) } @@ -266,13 +269,15 @@ func TestCreateOrUpdate_Paused(t *testing.T) { Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, + Paused: true, }, - Paused: true, }, } stsName := types.NamespacedName{Namespace: cr.Namespace, Name: "vmstorage-" + cr.Name} @@ -312,27 +317,29 @@ func TestCreateOrUpdate_LBDeploymentWithHPA(t *testing.T) { Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - Spec: vmv1beta1.VMAuthLoadBalancerSpec{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - }, - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(1)), - MaxReplicas: 5, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 5, + }, }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, }, }, } diff --git a/internal/controller/operator/factory/vmcluster/vmcluster_test.go b/internal/controller/operator/factory/vmcluster/vmcluster_test.go index 33f3d57dea..c3dd004471 100644 --- a/internal/controller/operator/factory/vmcluster/vmcluster_test.go +++ b/internal/controller/operator/factory/vmcluster/vmcluster_test.go @@ -13,6 +13,7 @@ import ( autoscalingv1 "k8s.io/api/autoscaling/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,6 +30,16 @@ import ( "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" ) +// containerArgs returns the Args of the named container in spec, or nil if absent. +func containerArgs(spec corev1.PodSpec, name string) []string { + for _, c := range spec.Containers { + if c.Name == name { + return c.Args + } + } + return nil +} + func TestCreateOrUpdate(t *testing.T) { type opts struct { cr *vmv1beta1.VMCluster @@ -70,32 +81,34 @@ func TestCreateOrUpdate(t *testing.T) { Labels: map[string]string{"label": "value"}, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(0)), - VMInsert: &vmv1beta1.VMInsert{ - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Annotations: map[string]string{"key": "value"}, - }, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0)), - }, - }, - VMStorage: &vmv1beta1.VMStorage{ - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Annotations: map[string]string{"key": "value"}, - Labels: map[string]string{"label": "value2"}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(0)), + VMInsert: &vmv1beta1.VMInsert{ + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Annotations: map[string]string{"key": "value"}, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0)), + }, }, - CommonAppsParams: vmv1beta1.CommonAppsParams{ + VMStorage: &vmv1beta1.VMStorage{ + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Annotations: map[string]string{"key": "value"}, + Labels: map[string]string{"label": "value2"}, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(2))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Annotations: map[string]string{"key": "value"}, + ReplicaCount: ptr.To(int32(2))}, }, - CommonAppsParams: vmv1beta1.CommonAppsParams{ + VMSelect: &vmv1beta1.VMSelect{ + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Annotations: map[string]string{"key": "value"}, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(2))}, + ReplicaCount: ptr.To(int32(2))}, + }, }, }, }, @@ -119,19 +132,21 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - InsertPorts: &vmv1beta1.InsertPorts{ - GraphitePort: "8025", - OpenTSDBHTTPPort: "3311", - InfluxPort: "5511", - }, - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(0)), - MaxReplicas: 3, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + InsertPorts: &vmv1beta1.InsertPorts{ + GraphitePort: "8025", + OpenTSDBHTTPPort: "3311", + InfluxPort: "5511", + }, + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(0)), + MaxReplicas: 3, + }, }, }, }, @@ -145,27 +160,29 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - InsertPorts: &vmv1beta1.InsertPorts{ - GraphitePort: "8025", - OpenTSDBHTTPPort: "3311", - InfluxPort: "5511", - }, - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(0)), - MaxReplicas: 3, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + InsertPorts: &vmv1beta1.InsertPorts{ + GraphitePort: "8025", + OpenTSDBHTTPPort: "3311", + InfluxPort: "5511", + }, + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(0)), + MaxReplicas: 3, + }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(0)), - MaxReplicas: 3, - Behaviour: &autoscalingv2.HorizontalPodAutoscalerBehavior{ - ScaleDown: &autoscalingv2.HPAScalingRules{}, + VMStorage: &vmv1beta1.VMStorage{ + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(0)), + MaxReplicas: 3, + Behaviour: &autoscalingv2.HorizontalPodAutoscalerBehavior{ + ScaleDown: &autoscalingv2.HPAScalingRules{}, + }, }, }, }, @@ -182,14 +199,16 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(2))}, - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(1)), - MaxReplicas: 3, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(2))}, + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 3, + }, }, }, }, @@ -214,21 +233,23 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - }, - VMStorage: &vmv1beta1.VMStorage{ - MaintenanceSelectNodeIDs: []int32{1, 3}, - MaintenanceInsertNodeIDs: []int32{0, 1, 2}, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(10))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(2))}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + }, + VMStorage: &vmv1beta1.VMStorage{ + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 1, 2}, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(10))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(2))}, + }, }, }, }, @@ -284,21 +305,23 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - }, - VMStorage: &vmv1beta1.VMStorage{ - MaintenanceSelectNodeIDs: []int32{1, 3}, - MaintenanceInsertNodeIDs: []int32{0, 1, 2}, - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(10))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(2))}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + }, + VMStorage: &vmv1beta1.VMStorage{ + MaintenanceSelectNodeIDs: []int32{1, 3}, + MaintenanceInsertNodeIDs: []int32{0, 1, 2}, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(10))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(2))}, + }, }, }, }, @@ -354,30 +377,32 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + }, + }, + VMSelect: &vmv1beta1.VMSelect{ CommonAppsParams: vmv1beta1.CommonAppsParams{ ReplicaCount: ptr.To(int32(0))}, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0))}, - InsertPorts: &vmv1beta1.InsertPorts{ - GraphitePort: "8025", - OpenTSDBHTTPPort: "3311", - InfluxPort: "5511", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0))}, + InsertPorts: &vmv1beta1.InsertPorts{ + GraphitePort: "8025", + OpenTSDBHTTPPort: "3311", + InfluxPort: "5511", + }, }, }, }, @@ -392,28 +417,30 @@ func TestCreateOrUpdate(t *testing.T) { Name: "cluster-1", }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "2", - ReplicationFactor: ptr.To(int32(2)), - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - Spec: vmv1beta1.VMAuthLoadBalancerSpec{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - }, - HPA: &vmv1beta1.EmbeddedHPA{ - MinReplicas: ptr.To(int32(1)), - MaxReplicas: 3, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "2", + ReplicationFactor: ptr.To(int32(2)), + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + HPA: &vmv1beta1.EmbeddedHPA{ + MinReplicas: ptr.To(int32(1)), + MaxReplicas: 3, + }, }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, }, }, }, @@ -433,33 +460,35 @@ func TestCreateOrUpdate(t *testing.T) { cr: &vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0)), - Volumes: []corev1.Volume{{ - Name: "vmselect-cachedir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/host/path/cache", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0)), + Volumes: []corev1.Volume{{ + Name: "vmselect-cachedir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/host/path/cache", + }, }, - }, - }}, - }, - CacheMountPath: "/cache", - VPA: &vmv1beta1.EmbeddedVPA{ - UpdatePolicy: &vpav1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpav1.UpdateModeRecreate), + }}, }, - ResourcePolicy: &vpav1.PodResourcePolicy{ - ContainerPolicies: []vpav1.ContainerResourcePolicy{ - { - ContainerName: "vmselect", - Mode: ptr.To(vpav1.ContainerScalingModeAuto), + CacheMountPath: "/cache", + VPA: &vmv1beta1.EmbeddedVPA{ + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpav1.UpdateModeRecreate), + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + { + ContainerName: "vmselect", + Mode: ptr.To(vpav1.ContainerScalingModeAuto), + }, }, }, - }, - Recommenders: []*vpav1.VerticalPodAutoscalerRecommenderSelector{ - {Name: "custom-recommender"}, + Recommenders: []*vpav1.VerticalPodAutoscalerRecommenderSelector{ + {Name: "custom-recommender"}, + }, }, }, }, @@ -531,25 +560,27 @@ func TestCreateOrUpdate(t *testing.T) { cr: &vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0)), - Volumes: []corev1.Volume{{ - Name: "vmstorage-db", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/host/path/storage", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0)), + Volumes: []corev1.Volume{{ + Name: "vmstorage-db", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/host/path/storage", + }, }, - }, - }}, - }, - VPA: &vmv1beta1.EmbeddedVPA{ - UpdatePolicy: &vpav1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpav1.UpdateModeInitial), + }}, }, - ResourcePolicy: &vpav1.PodResourcePolicy{ - ContainerPolicies: []vpav1.ContainerResourcePolicy{ - {ContainerName: "vmstorage"}, + VPA: &vmv1beta1.EmbeddedVPA{ + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpav1.UpdateModeInitial), + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + {ContainerName: "vmstorage"}, + }, }, }, }, @@ -636,17 +667,19 @@ func TestCreateOrUpdate(t *testing.T) { cr: &vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0)), - }, - VPA: &vmv1beta1.EmbeddedVPA{ - UpdatePolicy: &vpav1.PodUpdatePolicy{ - UpdateMode: ptr.To(vpav1.UpdateModeRecreate), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0)), }, - ResourcePolicy: &vpav1.PodResourcePolicy{ - ContainerPolicies: []vpav1.ContainerResourcePolicy{ - {ContainerName: "vminsert"}, + VPA: &vmv1beta1.EmbeddedVPA{ + UpdatePolicy: &vpav1.PodUpdatePolicy{ + UpdateMode: ptr.To(vpav1.UpdateModeRecreate), + }, + ResourcePolicy: &vpav1.PodResourcePolicy{ + ContainerPolicies: []vpav1.ContainerResourcePolicy{ + {ContainerName: "vminsert"}, + }, }, }, }, @@ -729,9 +762,11 @@ func TestCreateOrUpdate(t *testing.T) { Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(0)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(0)), + }, }, }, }, @@ -760,24 +795,82 @@ func TestCreateOrUpdate(t *testing.T) { Namespace: "default", }, Spec: vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ + Labels: map[string]string{"env": "prod"}, + Annotations: map[string]string{"controller": "true"}, }, }, - ManagedMetadata: &vmv1beta1.ManagedObjectsMetadata{ - Labels: map[string]string{"env": "prod"}, - Annotations: map[string]string{"controller": "true"}, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var set appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, set.Labels) + assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) + var svc corev1.Service + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) + assert.Equal(t, map[string]string{ + "env": "prod", + "app.kubernetes.io/name": "vmselect", + "app.kubernetes.io/instance": "base", + "app.kubernetes.io/component": "monitoring", + "app.kubernetes.io/part-of": "vmcluster", + "managed-by": "vm-operator", + }, svc.Labels) + }, + }) + + // common labels + f(opts{ + cfgMutator: func(c *config.BaseOperatorConf) { + c.CommonLabels = map[string]string{"env": "prod"} + c.CommonAnnotations = map[string]string{"controller": "true"} + }, + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "base", + Namespace: "default", + }, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, + }, }, }, }, @@ -803,61 +896,479 @@ func TestCreateOrUpdate(t *testing.T) { "app.kubernetes.io/part-of": "vmcluster", "managed-by": "vm-operator", }, svc.Labels) + }}) + + // pools: two pools with shared vminsert — pool STSes get pool label, instance label stays the cluster name, top-level vmstorage not created + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot"}, + {Name: "cold"}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + for _, poolName := range []string{"hot", "cold"} { + stsName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName) + var sts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: stsName}, &sts), "pool STS %s should exist", stsName) + // instance label must be the cluster name, not cluster-pool + assert.Equal(t, cr.Name, sts.Labels["app.kubernetes.io/instance"], "instance label for pool %s", poolName) + // STS selector must include the pool label so per-pool selectors are disjoint + assert.Equal(t, poolName, sts.Spec.Selector.MatchLabels["app.kubernetes.io/pool"], "selector pool label for pool %s", poolName) + // pod template must also carry the pool label so the STS selector matches its pods + assert.Equal(t, poolName, sts.Spec.Template.Labels["app.kubernetes.io/pool"], "pod template pool label for pool %s", poolName) + } + // top-level vmstorage must NOT be created when pools are defined + var topSts appsv1.StatefulSet + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, &topSts) + assert.True(t, k8serrors.IsNotFound(err), "top-level vmstorage STS must not exist when pools are defined") + // shared vminsert must still be created (no pool has a dedicated insert) + var dep appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, &dep)) + // shared vminsert must route to both pools' own storage nodes + args := containerArgs(dep.Spec.Template.Spec, "vminsert") + hotStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + coldStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "cold") + var hasHot, hasCold bool + for _, a := range args { + if !strings.HasPrefix(a, "-storageNode=") { + continue + } + hasHot = hasHot || strings.Contains(a, hotStorageName) + hasCold = hasCold || strings.Contains(a, coldStorageName) + } + assert.True(t, hasHot, "vminsert args should reference hot pool storage, got %v", args) + assert.True(t, hasCold, "vminsert args should reference cold pool storage, got %v", args) + }, + }) + + // pools: pool with dedicated vminsert — pool insert created, top-level vminsert skipped + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + // pool insert must exist with pool-scoped name + poolInsertName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, "hot") + var poolDep appsv1.Deployment + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: poolInsertName}, &poolDep)) + assert.Equal(t, cr.Name, poolDep.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", poolDep.Labels["app.kubernetes.io/pool"]) + // a pool's dedicated vminsert must route to its own storage nodes, not the + // (nonexistent) top-level storage StatefulSet + args := containerArgs(poolDep.Spec.Template.Spec, "vminsert") + hotStorageName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + baseStorageName := cr.PrefixedName(vmv1beta1.ClusterComponentStorage) + var hasHot bool + for _, a := range args { + if !strings.HasPrefix(a, "-storageNode=") { + continue + } + assert.False(t, strings.Contains(a, baseStorageName) && !strings.Contains(a, hotStorageName), + "pool vminsert must not reference the top-level storage name, got %q", a) + hasHot = hasHot || strings.Contains(a, hotStorageName) + } + assert.True(t, hasHot, "pool vminsert args should reference hot pool storage, got %v", args) + // top-level vminsert must NOT be created when any pool has its own insert + var topDep appsv1.Deployment + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, &topDep) + assert.True(t, k8serrors.IsNotFound(err), "top-level vminsert must not exist when a pool has a dedicated insert") + }, + }) + + // pools: switching an existing cluster to a pool with a dedicated vminsert must clean up + // the now-orphaned top-level vminsert Deployment/Service + { + clusterWithPoolInsert := &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + Status: vmv1beta1.VMClusterStatus{ + LastAppliedSpec: &vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + } + owner := clusterWithPoolInsert.AsOwner() + topInsertName := clusterWithPoolInsert.PrefixedName(vmv1beta1.ClusterComponentInsert) + insertSelectorLabels := clusterWithPoolInsert.SelectorLabels(vmv1beta1.ClusterComponentInsert) + svcLabels := labels.Merge(insertSelectorLabels, clusterWithPoolInsert.SelectorLabels(vmv1beta1.ClusterComponentCommon)) + f(opts{ + cr: clusterWithPoolInsert, + predefinedObjects: []runtime.Object{ + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: topInsertName, + Labels: insertSelectorLabels, + OwnerReferences: []metav1.OwnerReference{owner}, + }, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: topInsertName, + Labels: svcLabels, + OwnerReferences: []metav1.OwnerReference{owner}, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var topDep appsv1.Deployment + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: topInsertName}, &topDep) + assert.True(t, k8serrors.IsNotFound(err), "orphaned top-level vminsert Deployment must be removed once a pool has a dedicated insert") + var topSvc corev1.Service + err = rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: topInsertName}, &topSvc) + assert.True(t, k8serrors.IsNotFound(err), "orphaned top-level vminsert Service must be removed once a pool has a dedicated insert") + }, + }) + } + + // pools: NetworkPolicy for a pool's vmstorage/vminsert must be pool-scoped (name, selector, + // and labels), not collide with the base cluster's NetworkPolicy naming + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + NetworkPolicy: &vmv1beta1.EmbeddedNetworkPolicy{}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + NetworkPolicy: &vmv1beta1.EmbeddedNetworkPolicy{}, + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + storageNPName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot") + var storageNP networkingv1.NetworkPolicy + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: storageNPName}, &storageNP)) + assert.Equal(t, cr.Name, storageNP.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", storageNP.Labels["app.kubernetes.io/pool"]) + assert.Equal(t, "hot", storageNP.Spec.PodSelector.MatchLabels["app.kubernetes.io/pool"]) + + insertNPName := cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, "hot") + var insertNP networkingv1.NetworkPolicy + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: insertNPName}, &insertNP)) + assert.Equal(t, cr.Name, insertNP.Labels["app.kubernetes.io/instance"]) + assert.Equal(t, "hot", insertNP.Labels["app.kubernetes.io/pool"]) + assert.Equal(t, "hot", insertNP.Spec.PodSelector.MatchLabels["app.kubernetes.io/pool"]) + }, + }) + + // base (no pools): VMStorage.RetentionPeriod overrides cluster-level RetentionPeriod in the generated args + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + RetentionPeriod: "90d", + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var sts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, &sts)) + var storageArgs []string + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "vmstorage" { + storageArgs = c.Args + } + } + hasRetention90d := false + hasRetention1 := false + for _, a := range storageArgs { + if a == "-retentionPeriod=90d" { + hasRetention90d = true + } + if a == "-retentionPeriod=1" { + hasRetention1 = true + } + } + assert.True(t, hasRetention90d, "VMStorage.RetentionPeriod should override cluster-level: got args %v", storageArgs) + assert.False(t, hasRetention1, "cluster-level RetentionPeriod should be overridden: got args %v", storageArgs) + }, + }) + + // pools: a pool's VMStorage.RetentionPeriod overrides cluster-level RetentionPeriod for that + // pool's own StatefulSet only, independent of the base cluster-level value + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + RetentionPeriod: "90d", + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var poolSts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot")}, &poolSts)) + var poolArgs []string + for _, c := range poolSts.Spec.Template.Spec.Containers { + if c.Name == "vmstorage" { + poolArgs = c.Args + } + } + assert.Contains(t, poolArgs, "-retentionPeriod=90d", "pool vmstorage args: %v", poolArgs) + + var topSts appsv1.StatefulSet + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, &topSts) + assert.True(t, k8serrors.IsNotFound(err), "top-level vmstorage STS must not exist when pools are defined") + }, + }) + + // pools: a pool overriding only one field must inherit the cluster's actual (possibly + // customized) top-level values for every other field, not the generic defaults + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + StorageDataPath: "/custom/vmstorage-data", + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + RetentionPeriod: "90d", + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var poolSts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot")}, &poolSts)) + var poolArgs []string + for _, c := range poolSts.Spec.Template.Spec.Containers { + if c.Name == "vmstorage" { + poolArgs = c.Args + } + } + assert.Contains(t, poolArgs, "-retentionPeriod=90d", "pool vmstorage args: %v", poolArgs) + assert.Contains(t, poolArgs, "-storageDataPath=/custom/vmstorage-data", + "pool must inherit the cluster's customized storageDataPath, not the generic default: %v", poolArgs) + }, + }) + + // pools: a VMStorage topologySpreadConstraint with no labelSelector must default to a + // pool-scoped selector, not one matching every pool's vmstorage pods + f(opts{ + cr: &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "kubernetes.io/hostname"}, + }, + }, + }, + }, + { + Name: "cold", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + }, + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var poolSts appsv1.StatefulSet + assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, "hot")}, &poolSts)) + var tscs []corev1.TopologySpreadConstraint + tscs = append(tscs, poolSts.Spec.Template.Spec.TopologySpreadConstraints...) + if !assert.Len(t, tscs, 1) { + return + } + sel := tscs[0].LabelSelector + if !assert.NotNil(t, sel) { + return + } + assert.Equal(t, "hot", sel.MatchLabels["app.kubernetes.io/pool"], + "topologySpreadConstraint's default labelSelector must be scoped to this pool, got %v", sel.MatchLabels) }, }) - // common labels - f(opts{ - cfgMutator: func(c *config.BaseOperatorConf) { - c.CommonLabels = map[string]string{"env": "prod"} - c.CommonAnnotations = map[string]string{"controller": "true"} - }, - cr: &vmv1beta1.VMCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "base", - Namespace: "default", - }, - Spec: vmv1beta1.VMClusterSpec{ + // pools: a single dedicated-insert pool with requestsLoadBalancer enabled must not have its + // load-balanced internal Service deleted as an orphan on a steady-state reconcile + { + specWithLBPool := vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(0))}, }, }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + }, + } + clusterWithLBPool := &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "cluster-1"}, + Spec: specWithLBPool, + Status: vmv1beta1.VMClusterStatus{ + LastAppliedSpec: specWithLBPool.DeepCopy(), + }, + } + owner := clusterWithLBPool.AsOwner() + internalInsertName := clusterWithLBPool.PrefixedInternalName(vmv1beta1.ClusterComponentInsert) + insertSvcLabels := labels.Merge( + clusterWithLBPool.SelectorLabels(vmv1beta1.ClusterComponentInsert), + clusterWithLBPool.SelectorLabels(vmv1beta1.ClusterComponentCommon)) + f(opts{ + cr: clusterWithLBPool, + predefinedObjects: []runtime.Object{ + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: internalInsertName, + Labels: insertSvcLabels, + OwnerReferences: []metav1.OwnerReference{owner}, }, }, }, - }, - validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { - var set appsv1.StatefulSet - assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: "vmselect-base"}, &set)) - assert.Equal(t, map[string]string{ - "env": "prod", - "app.kubernetes.io/name": "vmselect", - "app.kubernetes.io/instance": "base", - "app.kubernetes.io/component": "monitoring", - "app.kubernetes.io/part-of": "vmcluster", - "managed-by": "vm-operator", - }, set.Labels) - assert.Equal(t, map[string]string{"controller": "true"}, set.Annotations) - var svc corev1.Service - assert.NoError(t, rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect)}, &svc)) - assert.Equal(t, map[string]string{ - "env": "prod", - "app.kubernetes.io/name": "vmselect", - "app.kubernetes.io/instance": "base", - "app.kubernetes.io/component": "monitoring", - "app.kubernetes.io/part-of": "vmcluster", - "managed-by": "vm-operator", - }, svc.Labels) - }}) + validate: func(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMCluster) { + var svc corev1.Service + err := rclient.Get(ctx, types.NamespacedName{Namespace: cr.Namespace, Name: internalInsertName}, &svc) + assert.NoError(t, err, "pool's load-balanced internal insert Service must not be removed as an orphan") + }, + }) + } } func TestCreatOrUpdateClusterServices(t *testing.T) { @@ -892,7 +1403,9 @@ func TestCreatOrUpdateClusterServices(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{}, + }, }, }, ` objectmeta: @@ -941,24 +1454,26 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{ - Key: ptr.To("test-key"), - }, - VMStorage: &vmv1beta1.VMStorage{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - PublishNotReadyAddresses: true, - Ports: []corev1.ServicePort{ - { - Name: "web-rpc", - Port: 8011, - TargetPort: intstr.FromInt(8011), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{ + Key: ptr.To("test-key"), + }, + VMStorage: &vmv1beta1.VMStorage{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + { + Name: "web-rpc", + Port: 8011, + TargetPort: intstr.FromInt(8011), + }, }, }, }, + VMBackup: &vmv1beta1.VMBackup{}, }, - VMBackup: &vmv1beta1.VMBackup{}, }, }, }, ` @@ -1018,10 +1533,12 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{}, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Port: "8352", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{}, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Port: "8352", + }, }, }, }, @@ -1104,9 +1621,11 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{}, - VMSelect: &vmv1beta1.VMSelect{CommonAppsParams: vmv1beta1.CommonAppsParams{Port: "8352"}, - ClusterNativePort: "8477", ServiceSpec: &vmv1beta1.AdditionalServiceSpec{Spec: corev1.ServiceSpec{Type: "LoadBalancer"}}}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{}, + VMSelect: &vmv1beta1.VMSelect{CommonAppsParams: vmv1beta1.CommonAppsParams{Port: "8352"}, + ClusterNativePort: "8477", ServiceSpec: &vmv1beta1.AdditionalServiceSpec{Spec: corev1.ServiceSpec{Type: "LoadBalancer"}}}, + }, }, }, ` objectmeta: @@ -1229,9 +1748,11 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - InsertPorts: &vmv1beta1.InsertPorts{ - OpenTSDBHTTPPort: "8087", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + InsertPorts: &vmv1beta1.InsertPorts{ + OpenTSDBHTTPPort: "8087", + }, }, }, }, @@ -1275,17 +1796,19 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - UseAsDefault: true, - Spec: corev1.ServiceSpec{ - ClusterIP: "None", - Type: "ClusterIP", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + UseAsDefault: true, + Spec: corev1.ServiceSpec{ + ClusterIP: "None", + Type: "ClusterIP", + }, + }, + ClusterNativePort: "8055", + InsertPorts: &vmv1beta1.InsertPorts{ + OpenTSDBHTTPPort: "8087", }, - }, - ClusterNativePort: "8055", - InsertPorts: &vmv1beta1.InsertPorts{ - OpenTSDBHTTPPort: "8087", }, }, }, @@ -1350,26 +1873,28 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - VMInsert: &vmv1beta1.VMInsert{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - UseAsDefault: true, - EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "incorrect-label", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMInsert: &vmv1beta1.VMInsert{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + UseAsDefault: true, + EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{ + "app.kubernetes.io/instance": "incorrect-label", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + }, }, - Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-type": "external", + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), }, }, - Spec: corev1.ServiceSpec{ - Type: "LoadBalancer", - LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), + ClusterNativePort: "8055", + InsertPorts: &vmv1beta1.InsertPorts{ + OpenTSDBHTTPPort: "8087", }, }, - ClusterNativePort: "8055", - InsertPorts: &vmv1beta1.InsertPorts{ - OpenTSDBHTTPPort: "8087", - }, }, }, }, ` @@ -1438,28 +1963,30 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - }, - VMInsert: &vmv1beta1.VMInsert{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "incorrect-label", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + }, + VMInsert: &vmv1beta1.VMInsert{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{ + "app.kubernetes.io/instance": "incorrect-label", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + }, }, - Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-type": "external", + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), }, }, - Spec: corev1.ServiceSpec{ - Type: "LoadBalancer", - LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), + ClusterNativePort: "8055", + InsertPorts: &vmv1beta1.InsertPorts{ + OpenTSDBHTTPPort: "8087", }, }, - ClusterNativePort: "8055", - InsertPorts: &vmv1beta1.InsertPorts{ - OpenTSDBHTTPPort: "8087", - }, }, }, }, ` @@ -1635,29 +2162,31 @@ spec: f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default-1"}, Spec: vmv1beta1.VMClusterSpec{ - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - }, - VMInsert: &vmv1beta1.VMInsert{ - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - UseAsDefault: true, - EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "incorrect-label", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + }, + VMInsert: &vmv1beta1.VMInsert{ + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + UseAsDefault: true, + EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{ + "app.kubernetes.io/instance": "incorrect-label", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + }, }, - Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-type": "external", + Spec: corev1.ServiceSpec{ + Type: "LoadBalancer", + LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), }, }, - Spec: corev1.ServiceSpec{ - Type: "LoadBalancer", - LoadBalancerClass: ptr.To("service.k8s.aws/nlb"), + ClusterNativePort: "8055", + InsertPorts: &vmv1beta1.InsertPorts{ + OpenTSDBHTTPPort: "8087", }, }, - ClusterNativePort: "8055", - InsertPorts: &vmv1beta1.InsertPorts{ - OpenTSDBHTTPPort: "8087", - }, }, }, }, ` @@ -1855,16 +2384,18 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{Key: licenseKey}, - Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, }, }, }, @@ -1880,14 +2411,16 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, }, }, }, @@ -1905,17 +2438,19 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{Key: licenseKey}, - Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: false}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: false}, + }, }, }, }, @@ -1932,20 +2467,22 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{Key: licenseKey}, - Discovery: &vmv1beta1.VMClusterDiscovery{ - Enabled: true, - Interval: "5s", - Filter: `vmstorage-test-[0-1]\.`, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{ + Enabled: true, + Interval: "5s", + Filter: `vmstorage-test-[0-1]\.`, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, }, }, }, @@ -1961,29 +2498,140 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { }, ) - // component-level discovery overrides interval and filter + // pools: vmselect gets pool-grouped storage nodes, shared vminsert gets plain addresses f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{Key: licenseKey}, - Discovery: &vmv1beta1.VMClusterDiscovery{ - Enabled: true, - Interval: "5s", - Filter: `vmstorage-test-[0-1]\.`, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(4))}, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}}}, + {Name: "cold", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}}}, }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + func(t *testing.T, args []string) { + // vmselect: pools are grouped into a single storageNode flag (poolName/addr entries) + var storageNodeArg string + for _, a := range args { + if strings.HasPrefix(a, "-storageNode=") { + storageNodeArg = a + break + } + } + assert.Contains(t, storageNodeArg, "hot/", "vmselect: expected hot pool storageNode, got %v", args) + assert.Contains(t, storageNodeArg, "cold/", "vmselect: expected cold pool storageNode, got %v", args) + // no top-level (ungrouped) storage node when pools are defined + for _, val := range strings.Split(strings.TrimPrefix(storageNodeArg, "-storageNode="), ",") { + assert.True(t, strings.Contains(val, "/"), "vmselect: expected all storageNodes to be pool-grouped, got %q", val) + } + }, + func(t *testing.T, args []string) { + // shared vminsert: pool storage nodes are plain addresses without pool prefix + var storageNodeArg string + for _, a := range args { + if strings.HasPrefix(a, "-storageNode=") { + storageNodeArg = a + break + } + } + assert.NotEmpty(t, storageNodeArg, "vminsert: expected storageNode flag, got %v", args) + assert.NotContains(t, storageNodeArg, "hot/", "vminsert: should not have pool-grouped addresses, got %v", args) + assert.NotContains(t, storageNodeArg, "cold/", "vminsert: should not have pool-grouped addresses, got %v", args) + }, + ) + + // pools + discovery: srv+ addresses stay pool-grouped, and discoveryInterval/filter + // must still be emitted (previously dropped entirely whenever pools were defined) + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, Discovery: &vmv1beta1.VMClusterDiscovery{ Enabled: true, - Interval: "10s", - Filter: `vmstorage-test-[2-3]\.`, + Interval: "5s", + Filter: `vmstorage-test-.*\.`, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + Pools: []vmv1beta1.VMClusterPool{ + {Name: "hot", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}}}, + {Name: "cold", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}}}, + }, + }, + }, + func(t *testing.T, args []string) { + var storageNodeArg string + for _, a := range args { + if strings.HasPrefix(a, "-storageNode=") { + storageNodeArg = a + break + } + } + assert.Contains(t, storageNodeArg, "hot/srv+", "vmselect: expected hot pool srv+ storageNode, got %v", args) + assert.Contains(t, storageNodeArg, "cold/srv+", "vmselect: expected cold pool srv+ storageNode, got %v", args) + assert.True(t, hasArg(args, "-storageNode.discoveryInterval=5s"), "vmselect: expected discoveryInterval flag even with pools, got %v", args) + assert.True(t, hasArg(args, `-storageNode.filter=vmstorage-test-.*\.`), "vmselect: expected filter flag even with pools, got %v", args) + }, + func(t *testing.T, args []string) { + var hasHot, hasCold bool + for _, a := range args { + if !strings.HasPrefix(a, "-storageNode=") { + continue + } + hasHot = hasHot || strings.Contains(a, "srv+vmstorage-test-hot") + hasCold = hasCold || strings.Contains(a, "srv+vmstorage-test-cold") + } + assert.True(t, hasHot, "vminsert: expected hot pool srv+ storageNode, got %v", args) + assert.True(t, hasCold, "vminsert: expected cold pool srv+ storageNode, got %v", args) + assert.True(t, hasArg(args, "-storageNode.discoveryInterval=5s"), "vminsert: expected discoveryInterval flag even with pools, got %v", args) + assert.True(t, hasArg(args, `-storageNode.filter=vmstorage-test-.*\.`), "vminsert: expected filter flag even with pools, got %v", args) + }, + ) + + // component-level discovery overrides interval and filter + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{ + Enabled: true, + Interval: "5s", + Filter: `vmstorage-test-[0-1]\.`, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(4))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + Discovery: &vmv1beta1.VMClusterDiscovery{ + Enabled: true, + Interval: "10s", + Filter: `vmstorage-test-[2-3]\.`, + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, }, }, }, @@ -2001,17 +2649,19 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - ExtraStorageNodes: []vmv1beta1.VMStorageNode{ - {Addr: "localhost:10101"}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + ExtraStorageNodes: []vmv1beta1.VMStorageNode{ + {Addr: "localhost:10101"}, + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, }, }, @@ -2037,19 +2687,21 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { f(&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, Spec: vmv1beta1.VMClusterSpec{ - License: &vmv1beta1.License{Key: licenseKey}, - Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, - ExtraStorageNodes: []vmv1beta1.VMStorageNode{ - {Addr: "localhost:10101"}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + License: &vmv1beta1.License{Key: licenseKey}, + Discovery: &vmv1beta1.VMClusterDiscovery{Enabled: true}, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + ExtraStorageNodes: []vmv1beta1.VMStorageNode{ + {Addr: "localhost:10101"}, + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, }, }, }, @@ -2070,3 +2722,77 @@ func TestVMClusterDiscoveryArgs(t *testing.T) { }, ) } + +func TestVMClusterReplicationFactorArgs(t *testing.T) { + f := func(cr *vmv1beta1.VMCluster, validateSelect func(t *testing.T, args []string)) { + t.Helper() + ctx := context.Background() + fclient := k8stools.GetTestClientWithObjects(nil) + build.AddDefaults(fclient.Scheme()) + fclient.Scheme().Default(cr) + assert.NoError(t, CreateOrUpdate(ctx, cr, fclient)) + + var sts appsv1.StatefulSet + assert.NoError(t, fclient.Get(ctx, types.NamespacedName{ + Namespace: cr.Namespace, + Name: cr.PrefixedName(vmv1beta1.ClusterComponentSelect), + }, &sts)) + var args []string + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "vmselect" { + args = c.Args + } + } + validateSelect(t, args) + } + + // globalReplicationFactor is rendered as-is on vmselect + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + GlobalReplicationFactor: ptr.To(int32(2)), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + }, + }, + func(t *testing.T, args []string) { + assert.Contains(t, args, "-globalReplicationFactor=2") + }, + ) + + // pool-specific replicationFactor is rendered as :, alongside the + // cluster-level bare replicationFactor for pools without their own override + f(&vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ReplicationFactor: ptr.To(int32(2)), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(2))}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + ReplicationFactor: ptr.To(int32(3)), + VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(3))}}, + }, + {Name: "cold", VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}}}, + }, + }, + }, + func(t *testing.T, args []string) { + assert.Contains(t, args, "-replicationFactor=hot:3", "expected pool-specific replicationFactor, got %v", args) + assert.Contains(t, args, "-replicationFactor=2", "expected cluster-level replicationFactor as fallback for pools without an override, got %v", args) + }, + ) +} diff --git a/internal/controller/operator/factory/vmdistributed/util_test.go b/internal/controller/operator/factory/vmdistributed/util_test.go index 0bd348728e..17e16b3ea7 100644 --- a/internal/controller/operator/factory/vmdistributed/util_test.go +++ b/internal/controller/operator/factory/vmdistributed/util_test.go @@ -22,43 +22,55 @@ func TestMergeSpecs(t *testing.T) { // zone-specific retention period f(&vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1d", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1d", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, }, }, &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "30d", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "30d", + }, }, "zone-a", &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "30d", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "30d", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, }, }) // %ZONE% templating f(&vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1d", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - NodeSelector: map[string]string{ - "topology.kubernetes.io/zone": "%ZONE%", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1d", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + NodeSelector: map[string]string{ + "topology.kubernetes.io/zone": "%ZONE%", + }, }, }, }, }, &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "30d", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "30d", + }, }, "zone-a", &vmv1beta1.VMClusterSpec{ - RetentionPeriod: "30d", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), - NodeSelector: map[string]string{ - "topology.kubernetes.io/zone": "zone-a", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "30d", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + NodeSelector: map[string]string{ + "topology.kubernetes.io/zone": "zone-a", + }, }, }, }, diff --git a/internal/controller/operator/factory/vmdistributed/vmdistributed_reconcile_test.go b/internal/controller/operator/factory/vmdistributed/vmdistributed_reconcile_test.go index b3860e790b..fdbe78e7de 100644 --- a/internal/controller/operator/factory/vmdistributed/vmdistributed_reconcile_test.go +++ b/internal/controller/operator/factory/vmdistributed/vmdistributed_reconcile_test.go @@ -127,7 +127,7 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { { Name: zoneName, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ @@ -194,7 +194,7 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { { Name: zoneName, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ @@ -252,7 +252,7 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { { Name: zoneName, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ @@ -326,7 +326,7 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { { Name: zoneName, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ @@ -401,7 +401,7 @@ func Test_CreateOrUpdate_Actions(t *testing.T) { }) zoneSpec := vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To(int32(1))}, @@ -584,7 +584,7 @@ func Test_CreateOrUpdate_Paused(t *testing.T) { { Name: "zone-1", VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMStorage: &vmv1beta1.VMStorage{ CommonAppsParams: vmv1beta1.CommonAppsParams{ diff --git a/internal/controller/operator/factory/vmdistributed/vmdistributed_test.go b/internal/controller/operator/factory/vmdistributed/vmdistributed_test.go index ecda127bab..b37290790e 100644 --- a/internal/controller/operator/factory/vmdistributed/vmdistributed_test.go +++ b/internal/controller/operator/factory/vmdistributed/vmdistributed_test.go @@ -47,20 +47,22 @@ func newVMCluster(name, namespace, version string, owner metav1.OwnerReference) OwnerReferences: []metav1.OwnerReference{owner}, }, Spec: vmv1beta1.VMClusterSpec{ - ClusterVersion: version, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + ClusterVersion: version, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(int32(1)), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(int32(1)), + }, }, }, }, @@ -131,7 +133,7 @@ func beforeEach(o opts) *testData { Name: name, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ Name: name, - Spec: vmCluster.Spec, + Spec: vmCluster.Spec.VMClusterSpecBase, }, VMAgent: vmv1alpha1.VMDistributedZoneAgent{ Name: name, diff --git a/internal/controller/operator/factory/vmdistributed/zone.go b/internal/controller/operator/factory/vmdistributed/zone.go index efbe6a9f1a..85350abe32 100644 --- a/internal/controller/operator/factory/vmdistributed/zone.go +++ b/internal/controller/operator/factory/vmdistributed/zone.go @@ -224,7 +224,7 @@ func buildVMClusterBackend(ctx context.Context, rclient client.Client, cr *vmv1a return vmBackend{}, false, fmt.Errorf("vmcluster.spec: %w", err) } prevClusterSpec := vmCluster.Spec - vmCluster.Spec = *vmClusterSpec + vmCluster.Spec = vmv1beta1.VMClusterSpec{VMClusterSpecBase: *vmClusterSpec, Pools: prevClusterSpec.Pools} rclient.Scheme().Default(&vmCluster) // for maintenance and read-only modes setting VMInsert replicas down to 0 to enable VMAgent buffering if (z.TrafficMode == vmv1alpha1.VMDistributedTrafficModeReadOnly || z.TrafficMode == vmv1alpha1.VMDistributedTrafficModeMaintenance) && vmCluster.Spec.VMInsert != nil { diff --git a/internal/controller/operator/factory/vmdistributed/zone_test.go b/internal/controller/operator/factory/vmdistributed/zone_test.go index 6785839ced..1ece1cc5fe 100644 --- a/internal/controller/operator/factory/vmdistributed/zone_test.go +++ b/internal/controller/operator/factory/vmdistributed/zone_test.go @@ -50,7 +50,7 @@ func TestGetZones(t *testing.T) { Spec: vmv1alpha1.VMDistributedSpec{ ZoneCommon: vmv1alpha1.VMDistributedZoneCommon{ VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ ClusterVersion: "v0.2.0", }, }, @@ -64,7 +64,7 @@ func TestGetZones(t *testing.T) { { VMCluster: vmv1alpha1.VMDistributedZoneCluster{ Name: "inline", - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ ClusterVersion: "v0.1.0", }, }, @@ -90,7 +90,7 @@ func TestGetZones(t *testing.T) { Spec: vmv1alpha1.VMDistributedSpec{ ZoneCommon: vmv1alpha1.VMDistributedZoneCommon{ VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ ClusterVersion: "v0.2.0", VMStorage: &vmv1beta1.VMStorage{ VMInsertPort: "9999", @@ -107,7 +107,7 @@ func TestGetZones(t *testing.T) { { VMCluster: vmv1alpha1.VMDistributedZoneCluster{ Name: "inline", - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ ClusterVersion: "v0.1.0", }, }, @@ -240,7 +240,7 @@ func TestWaitForEmptyPQ(t *testing.T) { // Pre-compute the expected backend URL for VMCluster named "test" in ns "default" backendURL := (&vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "default"}, - Spec: vmv1beta1.VMClusterSpec{VMInsert: &vmv1beta1.VMInsert{}}, + Spec: vmv1beta1.VMClusterSpec{VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{VMInsert: &vmv1beta1.VMInsert{}}}, }).GetRemoteWriteURL() type opts struct { @@ -307,7 +307,7 @@ func TestWaitForEmptyPQ(t *testing.T) { backend := &vmv1beta1.VMCluster{ ObjectMeta: objMeta, - Spec: vmv1beta1.VMClusterSpec{VMInsert: &vmv1beta1.VMInsert{}}, + Spec: vmv1beta1.VMClusterSpec{VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{VMInsert: &vmv1beta1.VMInsert{}}}, } zs := &zones{ httpClient: &http.Client{ diff --git a/internal/controller/operator/reconcile_and_track_status_test.go b/internal/controller/operator/reconcile_and_track_status_test.go index 769f5375f1..67ae6ebf5e 100644 --- a/internal/controller/operator/reconcile_and_track_status_test.go +++ b/internal/controller/operator/reconcile_and_track_status_test.go @@ -224,7 +224,9 @@ func TestReconcileAndTrackStatus(t *testing.T) { // in Expanding status when PVC resize is still in progress (waitForPVCReady // returns a retryable wait.Interrupted error). func TestVMClusterRemainsExpandingDuringPVCResize(t *testing.T) { - clusterSpec := vmv1beta1.VMClusterSpec{RetentionPeriod: "1d"} + clusterSpec := vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{RetentionPeriod: "1d"}, + } cluster := &vmv1beta1.VMCluster{ ObjectMeta: metav1.ObjectMeta{Name: "test-vmcluster", Namespace: "default"}, Spec: clusterSpec, diff --git a/internal/converter/converter.go b/internal/converter/converter.go index a3324b3b33..78d07bfc42 100644 --- a/internal/converter/converter.go +++ b/internal/converter/converter.go @@ -1,7 +1,9 @@ package converter import ( + "bytes" "crypto/sha256" + "encoding/json" "fmt" "io" "net/http" @@ -18,7 +20,6 @@ import ( vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" - "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" ) type VMSingleHelmValues struct { @@ -584,15 +585,27 @@ func FetchChartDefaults(chart string) ([]byte, error) { return data, nil } +// unmarshalYAMLPreservingNumbers decodes YAML into v via its JSON representation, using +// json.Number so large integers survive the round-trip instead of being rounded through float64. +func unmarshalYAMLPreservingNumbers(data []byte, v any) error { + jsonBytes, err := k8syaml.YAMLToJSON(data) + if err != nil { + return err + } + dec := json.NewDecoder(bytes.NewReader(jsonBytes)) + dec.UseNumber() + return dec.Decode(v) +} + // MergeValues deep-merges base and override YAML, with override taking precedence. func MergeValues(base, override []byte) ([]byte, error) { var baseMap map[string]any var overrideMap map[string]any - if err := k8syaml.Unmarshal(base, &baseMap); err != nil { + if err := unmarshalYAMLPreservingNumbers(base, &baseMap); err != nil { return nil, fmt.Errorf("cannot unmarshal base values: %w", err) } - if err := k8syaml.Unmarshal(override, &overrideMap); err != nil { + if err := unmarshalYAMLPreservingNumbers(override, &overrideMap); err != nil { return nil, fmt.Errorf("cannot unmarshal override values: %w", err) } @@ -600,7 +613,7 @@ func MergeValues(base, override []byte) ([]byte, error) { normalizeHeaderMaps(overrideMap) return k8syaml.Marshal(overrideMap) } - if err := build.MergeDeep(&baseMap, &overrideMap, false); err != nil { + if err := vmv1beta1.MergeDeep(&baseMap, &overrideMap, false); err != nil { return nil, fmt.Errorf("cannot merge values: %w", err) } normalizeHeaderMaps(baseMap) diff --git a/internal/converter/converter_test.go b/internal/converter/converter_test.go index b1016cd43c..af767f346c 100644 --- a/internal/converter/converter_test.go +++ b/internal/converter/converter_test.go @@ -1655,6 +1655,19 @@ func TestMergeValues(t *testing.T) { ) } +// TestMergeValues_LargeIntegerPrecision ensures integers beyond float64's exact range survive +// both the base/override merge and the final decode without being rounded. +func TestMergeValues_LargeIntegerPrecision(t *testing.T) { + got, err := MergeValues( + []byte("nodeID: 9007199254740993\nkeep: 1\n"), + []byte("replicaCount: 9223372036854775807\n"), + ) + require.NoError(t, err) + s := string(got) + assert.Contains(t, s, "9007199254740993") + assert.Contains(t, s, "9223372036854775807") +} + // TestMergeValues_VMAlertDatasourceHeaders reproduces #2398: vmalert's default values.yaml // ships `server.datasource.headers: {}` (a map), which used to fail to unmarshal into the // operator's HTTPAuth.Headers []string field once merged with a user's override. diff --git a/internal/podutil/util.go b/internal/podutil/util.go index 90c9d980fc..e2d9cc3040 100644 --- a/internal/podutil/util.go +++ b/internal/podutil/util.go @@ -15,7 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" vmv1alpha1 "github.com/VictoriaMetrics/operator/api/operator/v1alpha1" - "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/build" + vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/k8stools" "github.com/VictoriaMetrics/operator/internal/controller/operator/factory/logger" ) @@ -261,7 +261,7 @@ func MergeSpecs[T any](a, b *T, name string) (*T, error) { if err != nil { return nil, fmt.Errorf("failed to render spec: %w", err) } - if err := build.MergeDeep(merged, b, false); err != nil { + if err := vmv1beta1.MergeDeep(merged, b, false); err != nil { return nil, fmt.Errorf("failed to merge spec: %w", err) } return merged, nil diff --git a/internal/webhook/operator/v1beta1/vmcluster_webhook.go b/internal/webhook/operator/v1beta1/vmcluster_webhook.go index de2cdd5518..0e98f0c751 100644 --- a/internal/webhook/operator/v1beta1/vmcluster_webhook.go +++ b/internal/webhook/operator/v1beta1/vmcluster_webhook.go @@ -53,7 +53,7 @@ func (*VMClusterCustomValidator) ValidateCreate(ctx context.Context, obj *vmv1be warnings = append(warnings, "deprecated property is defined `spec.vmstorage.vmbackup.acceptEula`, use `spec.license.key` or `spec.license.keyRef` instead.") logger.WithContext(ctx).Info("deprecated property is defined `spec.vmstorage.vmbackup.acceptEula`, use `spec.license.key` or `spec.license.keyRef` instead.") } - warnings = append(warnings, build.WarnOpenShiftClusterSpec(&obj.Spec)...) + warnings = append(warnings, warnOpenShiftCluster(ctx, obj)...) return } @@ -70,10 +70,47 @@ func (*VMClusterCustomValidator) ValidateUpdate(ctx context.Context, _, newObj * warnings = append(warnings, "deprecated property is defined `spec.vmbackup.acceptEula`, use `spec.license.key` or `spec.license.keyRef` instead.") logger.WithContext(ctx).Info("deprecated property is defined `spec.vmbackup.acceptEula`, use `spec.license.key` or `spec.license.keyRef` instead.") } - warnings = append(warnings, build.WarnOpenShiftClusterSpec(&newObj.Spec)...) + warnings = append(warnings, warnOpenShiftCluster(ctx, newObj)...) return } +// warnOpenShiftCluster returns OpenShift security context warnings for a VMCluster. +func warnOpenShiftCluster(ctx context.Context, cr *vmv1beta1.VMCluster) admission.Warnings { + var warnings admission.Warnings + if cr.Spec.VMSelect != nil { + warnings = append(warnings, build.WarnOpenShiftSecurityContext(cr.Spec.VMSelect.SecurityContext)...) + } + if len(cr.Spec.Pools) == 0 && cr.Spec.VMStorage != nil { + warnings = append(warnings, build.WarnOpenShiftSecurityContext(cr.Spec.VMStorage.SecurityContext)...) + } + if !vmv1beta1.HasAnyPoolInsert(cr.Spec.Pools) && cr.Spec.VMInsert != nil { + warnings = append(warnings, build.WarnOpenShiftSecurityContext(cr.Spec.VMInsert.SecurityContext)...) + } + if cr.Spec.RequestsLoadBalancer.Enabled { + warnings = append(warnings, build.WarnOpenShiftSecurityContext(cr.Spec.RequestsLoadBalancer.Spec.SecurityContext)...) + } + poolWarnings, err := build.WarnOpenShiftClusterPools(cr) + if err != nil { + logger.WithContext(ctx).Error(err, "cannot resolve pool spec for OpenShift security context warnings") + } + warnings = append(warnings, poolWarnings...) + return dedupeWarnings(warnings) +} + +// dedupeWarnings drops repeated warning strings, keeping the first occurrence's order. +func dedupeWarnings(warnings admission.Warnings) admission.Warnings { + seen := make(map[string]struct{}, len(warnings)) + deduped := make(admission.Warnings, 0, len(warnings)) + for _, w := range warnings { + if _, ok := seen[w]; ok { + continue + } + seen[w] = struct{}{} + deduped = append(deduped, w) + } + return deduped +} + // ValidateDelete implements admission.Validator so a webhook will be registered for the type func (*VMClusterCustomValidator) ValidateDelete(_ context.Context, _ *vmv1beta1.VMCluster) (admission.Warnings, error) { return nil, nil diff --git a/test/e2e/upgrade/upgrade_test.go b/test/e2e/upgrade/upgrade_test.go index 74b1d15521..7dff0048a8 100644 --- a/test/e2e/upgrade/upgrade_test.go +++ b/test/e2e/upgrade/upgrade_test.go @@ -280,7 +280,7 @@ var ( }, } vmcluster = &vmv1beta1.VMCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpec{VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ Spec: vmv1beta1.VMAuthLoadBalancerSpec{ @@ -324,7 +324,7 @@ var ( TerminationGracePeriodSeconds: ptr.To(int64(1)), }, }, - }, + }}, } vmalertmanager = &vmv1beta1.VMAlertmanager{ Spec: vmv1beta1.VMAlertmanagerSpec{ @@ -371,7 +371,7 @@ var ( ReadyTimeout: &metav1.Duration{Duration: 2 * time.Minute}, UpdatePause: &metav1.Duration{Duration: 1 * time.Second}, VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: vmv1beta1.VMClusterSpec{ + Spec: vmv1beta1.VMClusterSpecBase{ RetentionPeriod: "1", VMSelect: &vmv1beta1.VMSelect{ CommonAppsParams: vmv1beta1.CommonAppsParams{ diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index fd4116e089..da40cb0257 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -125,10 +125,11 @@ func getRevisionHistoryLimit(ctx context.Context, rclient client.Client, name ty } type httpRequestOpts struct { - dstURL string - method string - expectedCode int - payload string + dstURL string + method string + expectedCode int + payload string + expectedBodyContains string } func expectHTTPRequestToSucceed(ctx context.Context, opts httpRequestOpts) { @@ -163,10 +164,16 @@ func expectHTTPRequestToSucceed(ctx context.Context, opts httpRequestOpts) { return err } defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %w", err) + } if resp.StatusCode != opts.expectedCode { - b, _ := io.ReadAll(resp.Body) return fmt.Errorf("unexpected status code: got %d, want %d, body: %s", resp.StatusCode, opts.expectedCode, b) } + if opts.expectedBodyContains != "" && !strings.Contains(string(b), opts.expectedBodyContains) { + return fmt.Errorf("response body does not contain %q, body: %s", opts.expectedBodyContains, b) + } return nil }, 60*time.Second).ShouldNot(HaveOccurred()) } diff --git a/test/e2e/vmcluster_test.go b/test/e2e/vmcluster_test.go index 148142416d..cc042eb0bf 100644 --- a/test/e2e/vmcluster_test.go +++ b/test/e2e/vmcluster_test.go @@ -10,6 +10,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -57,20 +58,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -135,7 +138,9 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Namespace: namespace, Name: nsn.Name, }, - Spec: vmv1beta1.VMClusterSpec{RetentionPeriod: "1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{RetentionPeriod: "1"}, + }, }, nil, ), Entry("with all components", "all-services", @@ -145,20 +150,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{CommonAppsParams: vmv1beta1.CommonAppsParams{ ReplicaCount: ptr.To[int32](1), }, - }, - VMInsert: &vmv1beta1.VMInsert{CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - }, + }, }, }, }, @@ -170,15 +177,17 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -190,15 +199,17 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -211,26 +222,28 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseStrictSecurity: ptr.To(true), - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseStrictSecurity: ptr.To(true), - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseStrictSecurity: ptr.To(true), - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseStrictSecurity: ptr.To(true), + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -262,51 +275,53 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), - SecurityContext: &vmv1beta1.SecurityContext{ - PodSecurityContext: &corev1.PodSecurityContext{ - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To(int64(65534)), - RunAsGroup: ptr.To(int64(65534)), - }, - ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ - Privileged: ptr.To(false), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + SecurityContext: &vmv1beta1.SecurityContext{ + PodSecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To(int64(65534)), + RunAsGroup: ptr.To(int64(65534)), + }, + ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ + Privileged: ptr.To(false), + }, }, }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), - SecurityContext: &vmv1beta1.SecurityContext{ - PodSecurityContext: &corev1.PodSecurityContext{ - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To(int64(65534)), - RunAsGroup: ptr.To(int64(65534)), - }, - ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ - Privileged: ptr.To(false), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + SecurityContext: &vmv1beta1.SecurityContext{ + PodSecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To(int64(65534)), + RunAsGroup: ptr.To(int64(65534)), + }, + ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ + Privileged: ptr.To(false), + }, }, }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - UseDefaultResources: ptr.To(false), - ReplicaCount: ptr.To[int32](1), - SecurityContext: &vmv1beta1.SecurityContext{ - PodSecurityContext: &corev1.PodSecurityContext{ - RunAsNonRoot: ptr.To(true), - RunAsUser: ptr.To(int64(65534)), - RunAsGroup: ptr.To(int64(65534)), - }, - ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ - Privileged: ptr.To(false), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + UseDefaultResources: ptr.To(false), + ReplicaCount: ptr.To[int32](1), + SecurityContext: &vmv1beta1.SecurityContext{ + PodSecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To(int64(65534)), + RunAsGroup: ptr.To(int64(65534)), + }, + ContainerSecurityContext: &vmv1beta1.ContainerSecurityContext{ + Privileged: ptr.To(false), + }, }, }, }, @@ -340,28 +355,30 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - ExtraArgs: map[string]string{ - "httpListenAddr.useProxyProtocol": "true", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + ExtraArgs: map[string]string{ + "httpListenAddr.useProxyProtocol": "true", + }, }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - ExtraArgs: map[string]string{ - "httpListenAddr.useProxyProtocol": "true", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + ExtraArgs: map[string]string{ + "httpListenAddr.useProxyProtocol": "true", + }, }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - ExtraArgs: map[string]string{ - "httpListenAddr.useProxyProtocol": "true", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + ExtraArgs: map[string]string{ + "httpListenAddr.useProxyProtocol": "true", + }, }, }, }, @@ -376,23 +393,25 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -407,6 +426,169 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: cr.PrefixedInternalName(vmv1beta1.ClusterComponentSelect)}, &svc)).ToNot(HaveOccurred()) }, ), + Entry("with pools sharing the top-level vminsert", "with-pools-shared-insert", + &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: nsn.Name, + }, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + RetentionPeriod: "7d", + }, + }, + { + Name: "cold", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + RetentionPeriod: "365d", + }, + }, + }, + }, + }, + func(cr *vmv1beta1.VMCluster) { + expectedRetention := map[string]string{"hot": "7d", "cold": "365d"} + for _, poolName := range []string{"hot", "cold"} { + var sts appsv1.StatefulSet + nss := types.NamespacedName{Namespace: namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName)} + Expect(k8sClient.Get(ctx, nss, &sts)).ToNot(HaveOccurred()) + Expect(sts.Spec.Template.Labels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + Expect(sts.Spec.Selector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + var storageContainer corev1.Container + for _, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "vmstorage" { + storageContainer = c + } + } + Expect(storageContainer.Args).To(ContainElement(fmt.Sprintf("-retentionPeriod=%s", expectedRetention[poolName]))) + var svc corev1.Service + Expect(k8sClient.Get(ctx, nss, &svc)).ToNot(HaveOccurred()) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + var poolDep appsv1.Deployment + poolInsertNss := types.NamespacedName{Namespace: namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, poolName)} + Expect(k8serrors.IsNotFound(k8sClient.Get(ctx, poolInsertNss, &poolDep))).To(BeTrue()) + } + waitResourceDeleted(ctx, + types.NamespacedName{Namespace: namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, + &appsv1.StatefulSetList{}) + var dep appsv1.Deployment + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, &dep)).ToNot(HaveOccurred()) + expectHTTPRequestToSucceed(ctx, httpRequestOpts{ + dstURL: fmt.Sprintf("http://%s.%s.svc:8480/insert/0/prometheus/api/v1/import/prometheus", cr.PrefixedName(vmv1beta1.ClusterComponentInsert), namespace), + payload: "pool_shared_insert_metric 123\n", + expectedCode: 204, + }) + expectHTTPRequestToSucceed(ctx, httpRequestOpts{ + dstURL: fmt.Sprintf("http://%s.%s.svc:8481/select/0/prometheus/api/v1/query?query=pool_shared_insert_metric", cr.PrefixedName(vmv1beta1.ClusterComponentSelect), namespace), + expectedBodyContains: "pool_shared_insert_metric", + }) + }, + ), + Entry("with pools defining dedicated vminserts", "with-pools-dedicated-insert", + &vmv1beta1.VMCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: nsn.Name, + }, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + }, + Pools: []vmv1beta1.VMClusterPool{ + { + Name: "hot", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + }, + { + Name: "cold", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + }, + }, + }, + }, + func(cr *vmv1beta1.VMCluster) { + for _, poolName := range []string{"hot", "cold"} { + var sts appsv1.StatefulSet + storageNss := types.NamespacedName{Namespace: namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentStorage, poolName)} + Expect(k8sClient.Get(ctx, storageNss, &sts)).ToNot(HaveOccurred()) + Expect(sts.Spec.Template.Labels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + Expect(sts.Spec.Selector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + + var dep appsv1.Deployment + insertNss := types.NamespacedName{Namespace: namespace, Name: cr.PoolPrefixedName(vmv1beta1.ClusterComponentInsert, poolName)} + Expect(k8sClient.Get(ctx, insertNss, &dep)).ToNot(HaveOccurred()) + Expect(dep.Spec.Template.Labels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + Expect(dep.Spec.Selector.MatchLabels).To(HaveKeyWithValue("app.kubernetes.io/pool", poolName)) + + metricName := fmt.Sprintf("pool_dedicated_insert_metric_%s", poolName) + expectHTTPRequestToSucceed(ctx, httpRequestOpts{ + dstURL: fmt.Sprintf("http://%s.%s.svc:8480/insert/0/prometheus/api/v1/import/prometheus", insertNss.Name, namespace), + payload: fmt.Sprintf("%s 123\n", metricName), + expectedCode: 204, + }) + expectHTTPRequestToSucceed(ctx, httpRequestOpts{ + dstURL: fmt.Sprintf("http://%s.%s.svc:8481/select/0/prometheus/api/v1/query?query=%s", cr.PrefixedName(vmv1beta1.ClusterComponentSelect), namespace, metricName), + expectedBodyContains: metricName, + }) + } + waitResourceDeleted(ctx, + types.NamespacedName{Namespace: namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentStorage)}, + &appsv1.StatefulSetList{}) + waitResourceDeleted(ctx, + types.NamespacedName{Namespace: namespace, Name: cr.PrefixedName(vmv1beta1.ClusterComponentInsert)}, + &appsv1.DeploymentList{}) + }, + ), ) }) Context("update", func() { @@ -431,20 +613,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(initialReplicas), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(initialReplicas), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(initialReplicas), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(initialReplicas), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To(initialReplicas), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To(initialReplicas), + }, }, }, }, @@ -558,20 +742,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by scaling select and storage replicas to 2", "storage-select-r-2", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -600,20 +786,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by adding vmbackupmanager to vmstorage ", "with-backup", true, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -673,20 +861,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by scaling storage and insert replicas to 2", "storage-insert-r-2", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -717,20 +907,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by changing storage revisionHistoryLimit to 2", "storage-revision-2", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -749,20 +941,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by adding clusterNative ports", "storage-native-r-2", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -792,20 +986,22 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by deleting select component", "select-delete", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -846,22 +1042,24 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { }, ), Entry("by deleting storage and insert components", "storage-insert-delete", false, - &vmv1beta1.VMCluster{ - Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + &vmv1beta1.VMCluster{ + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -923,39 +1121,41 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by deleting deleting and renaming additional services", "select-additional-svc", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - }, - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ - Name: "my-service-name", - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - TargetPort: intstr.FromInt(8431), - Name: "web-port", - Port: 8435, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ + Name: "my-service-name", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(8431), + Name: "web-port", + Port: 8435, + }, }, }, }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - }, - ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - TargetPort: intstr.FromInt(8431), - Name: "web-port", - Port: 8435, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + ServiceSpec: &vmv1beta1.AdditionalServiceSpec{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(8431), + Name: "web-port", + Port: 8435, + }, }, }, }, @@ -1020,21 +1220,23 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by adding imagePullSecret", "storage-image-pull-secret", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - ImagePullSecrets: nil, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + ImagePullSecrets: nil, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1069,23 +1271,25 @@ var _ = Describe("e2e vmcluster", Label("vm", "cluster", "vmcluster"), func() { Entry("by switching to vmauth loadbalancer", "with-load-balancing", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: false, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: false, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1183,23 +1387,25 @@ up{baz="bar"} 123 Entry("by switching partially to vmauth loadbalanacer", "with-partial-load-balancing", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: false, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: false, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1314,28 +1520,30 @@ up{baz="bar"} 123 Entry("by running with load-balancer and modify vmauth", "with-load-balancing-modify-auth", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ - Enabled: true, - Spec: vmv1beta1.VMAuthLoadBalancerSpec{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - Port: "8431", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{ + Enabled: true, + Spec: vmv1beta1.VMAuthLoadBalancerSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + Port: "8431", + }, }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](2), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](2), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1430,37 +1638,39 @@ up{baz="bar"} 123 Entry("by changing vmstorage PVC metadata", "vmstorage-pvc-meta", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{Enabled: true}, - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), - }, - Storage: &vmv1beta1.StorageSpec{ - VolumeClaimTemplate: vmv1beta1.EmbeddedPersistentVolumeClaim{ - EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{ - "label-name": "before", + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{Enabled: true}, + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + Storage: &vmv1beta1.StorageSpec{ + VolumeClaimTemplate: vmv1beta1.EmbeddedPersistentVolumeClaim{ + EmbeddedObjectMetadata: vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{ + "label-name": "before", + }, }, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - Resources: corev1.VolumeResourceRequirements{ - Requests: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceStorage: resource.MustParse("10Gi"), + Spec: corev1.PersistentVolumeClaimSpec{ + Resources: corev1.VolumeResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: resource.MustParse("10Gi"), + }, }, }, }, }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1484,21 +1694,23 @@ up{baz="bar"} 123 Entry("by changing annotations for created objects", "manage-annotations", false, &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{Enabled: true}, - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RequestsLoadBalancer: vmv1beta1.VMAuthLoadBalancer{Enabled: true}, + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1613,26 +1825,28 @@ up{baz="bar"} 123 Entry("configures vmstorage with MaxUnavailable 2", "maxunavailable-2-integer", &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](4), - }, - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{"version": "old"}, - }, - RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ - MaxUnavailable: ptr.To(intstr.FromInt32(2)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](4), + }, + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{"version": "old"}, + }, + RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ + MaxUnavailable: ptr.To(intstr.FromInt32(2)), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1669,26 +1883,28 @@ up{baz="bar"} 123 Entry("configures vmstorage with MaxUnavailable 100%", "maxunavailable-100-percent", &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](4), - }, - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{"version": "old"}, - }, - RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ - MaxUnavailable: ptr.To(intstr.FromString("100%")), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](4), + }, + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{"version": "old"}, + }, + RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ + MaxUnavailable: ptr.To(intstr.FromString("100%")), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1729,29 +1945,31 @@ up{baz="bar"} 123 Entry("configures vmstorage with MaxUnavailable 100% and ignored PDB limitation", "maxunavailable-100-percent-pdb", &vmv1beta1.VMCluster{ Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](4), - }, - PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ - Labels: map[string]string{"version": "old"}, - }, - RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ - MaxUnavailable: ptr.To(intstr.FromString("100%")), - }, - PodDisruptionBudget: &vmv1beta1.EmbeddedPodDisruptionBudgetSpec{ - MinAvailable: ptr.To(intstr.FromInt32(3)), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](4), + }, + PodMetadata: &vmv1beta1.EmbeddedObjectMetadata{ + Labels: map[string]string{"version": "old"}, + }, + RollingUpdateStrategyBehavior: &vmv1beta1.StatefulSetUpdateStrategyBehavior{ + MaxUnavailable: ptr.To(intstr.FromString("100%")), + }, + PodDisruptionBudget: &vmv1beta1.EmbeddedPodDisruptionBudgetSpec{ + MinAvailable: ptr.To(intstr.FromInt32(3)), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1825,15 +2043,17 @@ up{baz="bar"} 123 Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, }, }, } @@ -1850,20 +2070,22 @@ up{baz="bar"} 123 Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ - ReplicaCount: ptr.To[int32](1), + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, }, }, }, @@ -1892,15 +2114,17 @@ up{baz="bar"} 123 Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, }, }, } @@ -1928,16 +2152,18 @@ up{baz="bar"} 123 Name: nsn.Name, }, Spec: vmv1beta1.VMClusterSpec{ - RetentionPeriod: "1", - Paused: true, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + RetentionPeriod: "1", + Paused: true, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}, + }, }, }, } diff --git a/test/e2e/vmdistributed_test.go b/test/e2e/vmdistributed_test.go index e3e159f323..69b4c66079 100644 --- a/test/e2e/vmdistributed_test.go +++ b/test/e2e/vmdistributed_test.go @@ -30,14 +30,16 @@ func genVMClusterSpec(opts ...func(*vmv1beta1.VMClusterSpec)) vmv1beta1.VMCluste } s := vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{ - CommonAppsParams: commonAppsParams, - }, - VMInsert: &vmv1beta1.VMInsert{ - CommonAppsParams: noReplicas, - }, - VMStorage: &vmv1beta1.VMStorage{ - CommonAppsParams: commonAppsParams, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{ + CommonAppsParams: commonAppsParams, + }, + VMInsert: &vmv1beta1.VMInsert{ + CommonAppsParams: noReplicas, + }, + VMStorage: &vmv1beta1.VMStorage{ + CommonAppsParams: commonAppsParams, + }, }, } for _, opt := range opts { @@ -126,7 +128,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -214,7 +216,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } zs[i].Name = objMeta.Name vmClusterSpec := genVMClusterSpec(vmClusterFn[i]) - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmAgents[i] = &vmv1beta1.VMAgent{ @@ -300,7 +302,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } zs[i].Name = objMeta.Name vmClusterSpec := genVMClusterSpec() - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -381,7 +383,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { Name: fmt.Sprintf("%s-%d", nsn.Name, i+1), } zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = genVMClusterSpec(vmClusterFn[i]) + zs[i].VMCluster.Spec = genVMClusterSpec(vmClusterFn[i]).VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -418,7 +420,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { Name: "%ZONE%", Spec: genVMClusterSpec(func(s *vmv1beta1.VMClusterSpec) { s.RetentionPeriod = "60d" - }), + }).VMClusterSpecBase, }, }, Zones: zs, @@ -459,7 +461,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { Name: fmt.Sprintf("%s-%d", nsn.Name, i+1), } zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = genVMClusterSpec() + zs[i].VMCluster.Spec = genVMClusterSpec().VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ ObjectMeta: objMeta, @@ -536,7 +538,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { s.ClusterVersion = initialVersion }) zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -613,7 +615,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { s.ClusterVersion = initialVersion }) zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -679,7 +681,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { if err != nil { return err } - cr.Spec.Zones[0].VMCluster.Spec = vmv1beta1.VMClusterSpec{ + cr.Spec.Zones[0].VMCluster.Spec = vmv1beta1.VMClusterSpecBase{ VMInsert: cr.Spec.Zones[0].VMCluster.Spec.VMInsert, VMSelect: cr.Spec.Zones[0].VMCluster.Spec.VMSelect, VMStorage: &vmv1beta1.VMStorage{ @@ -736,7 +738,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -829,7 +831,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -891,9 +893,11 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { zonesCount := 2 clusterSpec := vmv1beta1.VMClusterSpec{ - VMSelect: &vmv1beta1.VMSelect{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, - VMInsert: &vmv1beta1.VMInsert{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, - VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{ + VMSelect: &vmv1beta1.VMSelect{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, + VMInsert: &vmv1beta1.VMInsert{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, + VMStorage: &vmv1beta1.VMStorage{CommonAppsParams: vmv1beta1.CommonAppsParams{ReplicaCount: ptr.To[int32](1)}}, + }, } zs := make([]vmv1alpha1.VMDistributedZone, zonesCount) @@ -906,7 +910,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } zs[i].Name = objMeta.Name zs[i].VMCluster.Name = objMeta.Name - zs[i].VMCluster.Spec = clusterSpec + zs[i].VMCluster.Spec = clusterSpec.VMClusterSpecBase zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ObjectMeta: objMeta, Spec: clusterSpec} vmAgents[i] = &vmv1beta1.VMAgent{ObjectMeta: objMeta, Spec: genVMAgentSpec()} @@ -1031,7 +1035,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { Zones: []vmv1alpha1.VMDistributedZone{ { VMCluster: vmv1alpha1.VMDistributedZoneCluster{ - Spec: genVMClusterSpec(), + Spec: genVMClusterSpec().VMClusterSpecBase, }, }, }, @@ -1053,7 +1057,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name } @@ -1127,7 +1131,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ @@ -1216,7 +1220,7 @@ var _ = Describe("e2e VMDistributed", Label("vm", "vmdistributed"), func() { } vmClusterSpec := genVMClusterSpec() zs[i].Name = objMeta.Name - zs[i].VMCluster.Spec = vmClusterSpec + zs[i].VMCluster.Spec = vmClusterSpec.VMClusterSpecBase zs[i].VMCluster.Name = objMeta.Name zs[i].VMAgent.Name = objMeta.Name vmClusters[i] = &vmv1beta1.VMCluster{ diff --git a/test/e2e/watchnamespace/controllers_test.go b/test/e2e/watchnamespace/controllers_test.go index 6c33c81610..5ccf089419 100644 --- a/test/e2e/watchnamespace/controllers_test.go +++ b/test/e2e/watchnamespace/controllers_test.go @@ -39,7 +39,9 @@ var _ = Describe("Controllers", func() { Namespace: namespace, Name: "test-vmcluster", }, - Spec: vmv1beta1.VMClusterSpec{RetentionPeriod: "1"}, + Spec: vmv1beta1.VMClusterSpec{ + VMClusterSpecBase: vmv1beta1.VMClusterSpecBase{RetentionPeriod: "1"}, + }, }, &vmv1beta1.VMAgent{ ObjectMeta: metav1.ObjectMeta{