Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions api/v1alpha2/etcdcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,31 @@ type EtcdClusterSpec struct {
// status.observed.
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

// ImageRepository overrides the operator-wide etcd image repository
// (--etcd-image-repository) for this cluster: registry host and path,
// without a tag or digest. The tag stays v<spec.version> plus
// ImageTagSuffix, so the image always runs the version the operator
// manages.
//
// Latched through status.observed and copied onto every member; a
// running member switches to it when its Pod is recreated.
// +kubebuilder:validation:MaxLength=255
// +kubebuilder:validation:XValidation:rule="!self.contains('@') && !self.substring(self.lastIndexOf('/') + 1).contains(':')",message="spec.imageRepository is a repository without a tag or digest; the tag is v<spec.version> followed by spec.imageTagSuffix"
// +optional
ImageRepository string `json:"imageRepository,omitempty"`

// ImageTagSuffix is appended to the v<spec.version> image tag, for
// images published with a suffix such as -fips or the version of an
// etcd wrapper. It starts with '-' so the tag still begins with the
// etcd version.
//
// Latched through status.observed and copied onto every member; a
// running member switches to it when its Pod is recreated.
// +kubebuilder:validation:MaxLength=100
// +kubebuilder:validation:XValidation:rule="self.matches('^-[A-Za-z0-9_.-]+$')",message="spec.imageTagSuffix starts with '-' and contains only letters, digits, '_', '.' and '-', so the image tag still begins with v<spec.version>"
// +optional
ImageTagSuffix string `json:"imageTagSuffix,omitempty"`
}

// AdditionalMetadata is a set of labels and annotations the operator merges
Expand Down Expand Up @@ -683,6 +708,14 @@ type ObservedClusterSpec struct {
// reached.
// +optional
ExtraArgs map[string]string `json:"extraArgs,omitempty"`

// ImageRepository is the locked target etcd image repository override.
// +optional
ImageRepository string `json:"imageRepository,omitempty"`

// ImageTagSuffix is the locked target etcd image tag suffix.
// +optional
ImageTagSuffix string `json:"imageTagSuffix,omitempty"`
}

// EtcdClusterStatus defines the observed state of an etcd cluster.
Expand Down
12 changes: 12 additions & 0 deletions api/v1alpha2/etcdmember_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,18 @@ type EtcdMemberSpec struct {
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`

// ImageRepository mirrors the latched EtcdCluster.spec.imageRepository.
// The cluster controller copies it onto each member at creation and
// keeps it in sync afterwards; a running Pod picks a change up only
// when it is recreated.
// +optional
ImageRepository string `json:"imageRepository,omitempty"`

// ImageTagSuffix mirrors the latched EtcdCluster.spec.imageTagSuffix,
// with the same sync rule as ImageRepository.
// +optional
ImageTagSuffix string `json:"imageTagSuffix,omitempty"`

// Bootstrap indicates this member is part of the initial cluster formation.
// When true the member starts with --initial-cluster-state=new.
// +optional
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1259,6 +1259,38 @@ spec:
type: object
x-kubernetes-map-type: atomic
type: array
imageRepository:
description: |-
ImageRepository overrides the operator-wide etcd image repository
(--etcd-image-repository) for this cluster: registry host and path,
without a tag or digest. The tag stays v<spec.version> plus
ImageTagSuffix, so the image always runs the version the operator
manages.

Latched through status.observed and copied onto every member; a
running member switches to it when its Pod is recreated.
maxLength: 255
type: string
x-kubernetes-validations:
- message: spec.imageRepository is a repository without a tag or digest;
the tag is v<spec.version> followed by spec.imageTagSuffix
rule: '!self.contains(''@'') && !self.substring(self.lastIndexOf(''/'')
+ 1).contains('':'')'
imageTagSuffix:
description: |-
ImageTagSuffix is appended to the v<spec.version> image tag, for
images published with a suffix such as -fips or the version of an
etcd wrapper. It starts with '-' so the tag still begins with the
etcd version.

Latched through status.observed and copied onto every member; a
running member switches to it when its Pod is recreated.
maxLength: 100
type: string
x-kubernetes-validations:
- message: spec.imageTagSuffix starts with '-' and contains only letters,
digits, '_', '.' and '-', so the image tag still begins with v<spec.version>
rule: self.matches('^-[A-Za-z0-9_.-]+$')
options:
description: |-
Options carries etcd server tuning flags (backend quota,
Expand Down Expand Up @@ -2978,6 +3010,14 @@ spec:
type: object
x-kubernetes-map-type: atomic
type: array
imageRepository:
description: ImageRepository is the locked target etcd image repository
override.
type: string
imageTagSuffix:
description: ImageTagSuffix is the locked target etcd image tag
suffix.
type: string
options:
description: |-
Options is the locked target etcd tuning flags for member Pods.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1056,6 +1056,18 @@ spec:
type: object
x-kubernetes-map-type: atomic
type: array
imageRepository:
description: |-
ImageRepository mirrors the latched EtcdCluster.spec.imageRepository.
The cluster controller copies it onto each member at creation and
keeps it in sync afterwards; a running Pod picks a change up only
when it is recreated.
type: string
imageTagSuffix:
description: |-
ImageTagSuffix mirrors the latched EtcdCluster.spec.imageTagSuffix,
with the same sync rule as ImageRepository.
type: string
initialCluster:
description: |-
InitialCluster is the value passed to etcd's --initial-cluster flag.
Expand Down
19 changes: 16 additions & 3 deletions controllers/etcdcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,8 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// ── Member templates ─────────────────────────────────────────────────────

// syncMemberTemplates copies the latched Pod-template fields (options,
// extraArgs, resources) onto existing members. Running Pods are left alone;
// extraArgs, resources, image repository and tag suffix) onto existing
// members. Running Pods are left alone;
// a Pod recreated for any reason, including a deliberate delete to roll a
// change, picks the new template up. Storage, version, scheduling and
// metadata stay as they were at member creation: they are bound to the
Expand All @@ -420,13 +421,17 @@ func (r *EtcdClusterReconciler) syncMemberTemplates(ctx context.Context, cluster
m := &members[i]
if equality.Semantic.DeepEqual(m.Spec.Options, o.Options) &&
equality.Semantic.DeepEqual(m.Spec.ExtraArgs, o.ExtraArgs) &&
equality.Semantic.DeepEqual(m.Spec.Resources, o.Resources) {
equality.Semantic.DeepEqual(m.Spec.Resources, o.Resources) &&
m.Spec.ImageRepository == o.ImageRepository &&
m.Spec.ImageTagSuffix == o.ImageTagSuffix {
continue
}
patch := client.MergeFrom(m.DeepCopy())
m.Spec.Options = o.Options.DeepCopy()
m.Spec.ExtraArgs = maps.Clone(o.ExtraArgs)
m.Spec.Resources = *o.Resources.DeepCopy()
m.Spec.ImageRepository = o.ImageRepository
m.Spec.ImageTagSuffix = o.ImageTagSuffix
if err := r.Patch(ctx, m, patch); err != nil {
return fmt.Errorf("sync template of member %s: %w", m.Name, err)
}
Expand Down Expand Up @@ -488,6 +493,8 @@ func (r *EtcdClusterReconciler) bootstrap(
Options: cluster.Status.Observed.Options,
ExtraArgs: cluster.Status.Observed.ExtraArgs,
ImagePullSecrets: cluster.Status.Observed.ImagePullSecrets,
ImageRepository: cluster.Status.Observed.ImageRepository,
ImageTagSuffix: cluster.Status.Observed.ImageTagSuffix,
Bootstrap: true,
ClusterToken: cluster.Status.ClusterToken,
TLS: deriveMemberTLS(cluster),
Expand Down Expand Up @@ -894,6 +901,8 @@ func (r *EtcdClusterReconciler) scaleUp(
Options: cluster.Status.Observed.Options,
ExtraArgs: cluster.Status.Observed.ExtraArgs,
ImagePullSecrets: cluster.Status.Observed.ImagePullSecrets,
ImageRepository: cluster.Status.Observed.ImageRepository,
ImageTagSuffix: cluster.Status.Observed.ImageTagSuffix,
Bootstrap: false,
ClusterToken: cluster.Status.ClusterToken,
TLS: deriveMemberTLS(cluster),
Expand Down Expand Up @@ -2180,6 +2189,8 @@ func snapshotSpecIntoObserved(cluster *lll.EtcdCluster) {
Options: cluster.Spec.Options,
ImagePullSecrets: cluster.Spec.ImagePullSecrets,
ExtraArgs: cluster.Spec.ExtraArgs,
ImageRepository: cluster.Spec.ImageRepository,
ImageTagSuffix: cluster.Spec.ImageTagSuffix,
}
}

Expand All @@ -2202,7 +2213,9 @@ func specEqualsObserved(cluster *lll.EtcdCluster) bool {
equality.Semantic.DeepEqual(o.AdditionalMetadata, cluster.Spec.AdditionalMetadata) &&
equality.Semantic.DeepEqual(o.Options, cluster.Spec.Options) &&
equality.Semantic.DeepEqual(o.ImagePullSecrets, cluster.Spec.ImagePullSecrets) &&
equality.Semantic.DeepEqual(o.ExtraArgs, cluster.Spec.ExtraArgs)
equality.Semantic.DeepEqual(o.ExtraArgs, cluster.Spec.ExtraArgs) &&
o.ImageRepository == cluster.Spec.ImageRepository &&
o.ImageTagSuffix == cluster.Spec.ImageTagSuffix
}

// observedAdditionalMetadata returns the latched additionalMetadata target
Expand Down
21 changes: 13 additions & 8 deletions controllers/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,18 +122,23 @@ func memberDataDir(member *lll.EtcdMember) string {
return path.Join(etcdDataDirRoot, sub)
}

// resolveEtcdImage resolves a member's etcd container image from the
// operator-wide repository default (defaultRepo, from the operator's
// --etcd-image-repository flag) or the EtcdImage built-in when that is empty,
// tagged with "v"+spec.version. The operator keys every version-dependent
// behaviour off spec.version, so the image is always pinned to that version —
// there is no per-cluster tag override that could disagree with it.
// resolveEtcdImage resolves a member's etcd container image. The repository
// is the cluster's spec.imageRepository, else the operator-wide default
// (defaultRepo, from the operator's --etcd-image-repository flag), else the
// EtcdImage built-in. The tag is "v"+spec.version followed by the cluster's
// spec.imageTagSuffix. The operator keys every version-dependent behaviour
// off spec.version, so the tag always begins with that version — admission
// rejects a repository carrying its own tag and a suffix not starting with
// '-'.
func resolveEtcdImage(member *lll.EtcdMember, defaultRepo string) string {
repo := defaultRepo
repo := member.Spec.ImageRepository
if repo == "" {
repo = defaultRepo
}
if repo == "" {
repo = EtcdImage
}
return repo + ":v" + member.Spec.Version
return repo + ":v" + member.Spec.Version + member.Spec.ImageTagSuffix
}

// peerURL returns the etcd peer URL for a member, using the headless Service
Expand Down
91 changes: 91 additions & 0 deletions controllers/image_envtest_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package controllers

import (
"context"
"os"
"path/filepath"
"strings"
"testing"

"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"

lll "github.com/cozystack/etcd-operator/api/v1alpha2"
)

// TestImageAdmission_Envtest drives spec.imageRepository and
// spec.imageTagSuffix through a real apiserver with the generated CRDs, so
// the CEL rules that keep the tag pinned to spec.version are exercised as
// shipped. `make test` sets KUBEBUILDER_ASSETS; without it the test skips.
func TestImageAdmission_Envtest(t *testing.T) {
if os.Getenv("KUBEBUILDER_ASSETS") == "" {
t.Skip("KUBEBUILDER_ASSETS not set; envtest harness not initialized")
}
env := &envtest.Environment{
Scheme: testScheme(t),
CRDDirectoryPaths: []string{filepath.Join("..", "charts", "etcd-operator", "crd-bases")},
ErrorIfCRDPathMissing: true,
}
cfg, err := env.Start()
if err != nil {
t.Fatalf("envtest start: %v", err)
}
t.Cleanup(func() { _ = env.Stop() })
k8s, err := client.New(cfg, client.Options{Scheme: testScheme(t)})
if err != nil {
t.Fatalf("client: %v", err)
}
ctx := context.Background()

create := func(repo, suffix string) error {
c := &lll.EtcdCluster{
ObjectMeta: metav1.ObjectMeta{GenerateName: "image-", Namespace: "default"},
Spec: lll.EtcdClusterSpec{
Replicas: ptrInt32(3),
Version: "3.6.1",
Storage: lll.StorageSpec{Size: resource.MustParse("128Mi")},
ImageRepository: repo,
ImageTagSuffix: suffix,
},
}
err := k8s.Create(ctx, c)
if err == nil {
_ = k8s.Delete(ctx, c)
}
return err
}

for _, tc := range []struct{ repo, suffix string }{
{"ghcr.io/aenix-io/threek", ""},
{"registry.internal:5000/mirror/etcd", ""},
{"etcd", ""},
{"", "-threek.0.1.0"},
{"ghcr.io/aenix-io/threek", "-fips_1.2"},
} {
t.Run("accepts "+tc.repo+" "+tc.suffix, func(t *testing.T) {
if err := create(tc.repo, tc.suffix); err != nil {
t.Fatalf("apiserver rejected repository %q suffix %q: %v", tc.repo, tc.suffix, err)
}
})
}

for _, repo := range []string{"ghcr.io/aenix-io/threek:v3.6.1", "registry.internal:5000/etcd:latest", "ghcr.io/aenix-io/threek@sha256:0123"} {
t.Run("rejects repository "+repo, func(t *testing.T) {
err := create(repo, "")
if err == nil || !strings.Contains(err.Error(), "spec.imageRepository is a repository without a tag or digest") {
t.Fatalf("repository %q: error = %v, want the repository rule", repo, err)
}
})
}

for _, suffix := range []string{"threek", "-", "-a/b", "-a:b", "-a b"} {
t.Run("rejects suffix "+suffix, func(t *testing.T) {
err := create("", suffix)
if err == nil || !strings.Contains(err.Error(), "spec.imageTagSuffix starts with '-'") {
t.Fatalf("suffix %q: error = %v, want the suffix rule", suffix, err)
}
})
}
}
Loading
Loading